tests pass; still need catalog

Closes #639 
Closes #638

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.9
+current_version = 0.4.0
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True

.github/ISSUE_TEMPLATE/bug-node.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Node / Typescript
+description: File a bug report
+title: "bug(node): "
+labels: [bug, typescript]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `npm list | grep vectordb`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/bug-python.yml

@@ -0,0 +1,33 @@
+name: Bug Report - Python
+description: File a bug report
+title: "bug(python): "
+labels: [bug, python]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to fill out this bug report!
+  - type: input
+    id: version
+    attributes:
+      label: LanceDB version
+      description: What version of LanceDB are you using? `python -c "import lancedb; print(lancedb.__version__)"`.
+      placeholder: v0.3.2
+    validations:
+      required: false
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Are there known steps to reproduce?
+      description: |
+        Let us know how to reproduce the bug and we may be able to fix it more
+        quickly. This is not required, but it is helpful.
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Discord Community Support
+    url: https://discord.com/invite/zMM32dvNtd
+    about: Please ask and answer questions here.

.github/ISSUE_TEMPLATE/documentation.yml

@@ -0,0 +1,23 @@
+name: 'Documentation improvement'
+description: Report an issue with the documentation.
+labels: [documentation]
+
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: >
+        Describe the issue with the documentation and how it can be fixed or improved.
+    validations:
+      required: true
+
+  - type: input
+    id: link
+    attributes:
+      label: Link
+      description: >
+        Provide a link to the existing documentation, if applicable.
+      placeholder: ex. https://lancedb.github.io/lancedb/guides/tables/...
+    validations:
+      required: false

.github/ISSUE_TEMPLATE/feature.yml

@@ -0,0 +1,31 @@
+name: Feature suggestion
+description: Suggestion a new feature for LanceDB
+title: "Feature: "
+labels: [enhancement]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Share a new idea for a feature or improvement. Be sure to search existing
+        issues first to avoid duplicates.
+  - type: dropdown
+    id: sdk
+    attributes:
+      label: SDK
+      description: Which SDK are you using? This helps us prioritize.
+      options:
+        - Python
+        - Node
+        - Rust
+      default: 0
+    validations:
+      required: false
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      description: |
+        Describe the feature and why it would be useful. If applicable, consider
+        providing a code example of what it might be like to use the feature.
+    validations:
+      required: true

.github/workflows/npm-publish.yml

@@ -37,8 +37,16 @@ jobs:
           path: |
             node/vectordb-*.tgz
 
-  node-macos-x86:
-    runs-on: macos-13
+  node-macos:
+    strategy:
+      matrix:
+        config:
+          - arch: x86_64-apple-darwin
+            runner: macos-13
+          - arch: aarch64-apple-darwin
+            # xlarge is implicitly arm64.
+            runner: macos-13-xlarge
+    runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
     steps:
@@ -51,35 +59,14 @@ jobs:
           cd node
           npm ci
       - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh x86_64-apple-darwin
+        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
       - name: Upload Darwin Artifacts
         uses: actions/upload-artifact@v3
         with:
           name: native-darwin
           path: |
             node/dist/lancedb-vectordb-darwin*.tgz
-
-  node-macos-arm64:
-    runs-on: macos-13-xlarge
-    # Only runs on tags that matches the make-release action
-    if: startsWith(github.ref, 'refs/tags/v')
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install system dependencies
-        run: brew install protobuf
-      - name: Install npm dependencies
-        run: |
-          cd node
-          npm ci
-      - name: Build MacOS native node modules
-        run: bash ci/build_macos_artifacts.sh aarch64-apple-darwin
-      - name: Upload Darwin Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: native-darwin
-          path: |
-            node/dist/lancedb-vectordb-darwin*.tgz            
+         
 
   node-linux:
     name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu

.github/workflows/python.yml

@@ -44,12 +44,19 @@ jobs:
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest
       run: pytest --doctest-modules lancedb
-  mac:
+  platform:
+    name: "Platform: ${{ matrix.config.name }}"
     timeout-minutes: 30
     strategy:
       matrix:
-        mac-runner: [ "macos-13", "macos-13-xlarge" ]
-    runs-on: "${{ matrix.mac-runner }}"
+        config: 
+          - name: x86 Mac
+            runner: macos-13
+          - name: Arm Mac
+            runner: macos-13-xlarge
+          - name: x86 Windows
+            runner: windows-latest
+    runs-on: "${{ matrix.config.runner }}"
     defaults:
       run:
         shell: bash
@@ -91,11 +98,7 @@ jobs:
         pip install "pydantic<2"
         pip install -e .[tests]
         pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest pytest-mock
     - name: Run tests
       run: pytest -m "not slow" -x -v --durations=30 tests
     - name: doctest

.github/workflows/rust.yml

@@ -24,6 +24,29 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
+  lint:
+    timeout-minutes: 30
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+        working-directory: rust
+    steps:
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0
+          lfs: true
+    - uses: Swatinem/rust-cache@v2
+      with:
+        workspaces: rust
+    - name: Install dependencies
+      run: |
+          sudo apt update
+          sudo apt install -y protobuf-compiler libssl-dev
+    - name: Run format
+      run: cargo fmt --all -- --check
+    - name: Run clippy
+      run: cargo clippy --all --all-features -- -D warnings
   linux:
     timeout-minutes: 30
     runs-on: ubuntu-22.04

Cargo.toml

@@ -5,24 +5,24 @@ exclude = ["python"]
 resolver = "2"
 
 [workspace.dependencies]
-lance = { "version" = "=0.8.20", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.8.20" }
-lance-linalg = { "version" = "=0.8.20" }
-lance-testing = { "version" = "=0.8.20" }
+lance = { "version" = "=0.9.1", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.9.1" }
+lance-linalg = { "version" = "=0.9.1" }
+lance-testing = { "version" = "=0.9.1" }
 # Note that this one does not include pyarrow
-arrow = { version = "47.0.0", optional = false }
-arrow-array = "47.0"
-arrow-data = "47.0"
-arrow-ipc = "47.0"
-arrow-ord = "47.0"
-arrow-schema = "47.0"
-arrow-arith = "47.0"
-arrow-cast = "47.0"
+arrow = { version = "49.0.0", optional = false }
+arrow-array = "49.0"
+arrow-data = "49.0"
+arrow-ipc = "49.0"
+arrow-ord = "49.0"
+arrow-schema = "49.0"
+arrow-arith = "49.0"
+arrow-cast = "49.0"
 chrono = "0.4.23"
 half = { "version" = "=2.3.1", default-features = false, features = [
     "num-traits",
 ] }
 log = "0.4"
-object_store = "0.7.1"
+object_store = "0.8.0"
 snafu = "0.7.4"
 url = "2"

docs/requirements.txt

@@ -2,3 +2,4 @@ mkdocs==1.4.2
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.1.3
 mkdocstrings[python]==0.20.0
+pydantic

docs/src/basic.md

@@ -64,18 +64,26 @@ We'll cover the basics of using LanceDB on your local machine in this section.
       tbl = db.create_table("table_from_df", data=df)
       ```
 
+      !!! warning
+
+            If the table already exists, LanceDB will raise an error by default.
+            If you want to overwrite the table, you can pass in `mode="overwrite"`
+            to the `createTable` function.
+
 === "Javascript"
       ```javascript
-      const tb = await db.createTable("my_table",
-                        data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                              {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      const tb = await db.createTable(
+        "myTable",
+        [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+         {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
       ```
-      
-!!! warning
 
-      If the table already exists, LanceDB will raise an error by default.
-      If you want to overwrite the table, you can pass in `mode="overwrite"`
-      to the `createTable` function.
+      !!! warning
+
+            If the table already exists, LanceDB will raise an error by default.
+            If you want to overwrite the table, you can pass in `"overwrite"`
+            to the `createTable` function like this: `await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })`
+      
 
 ??? info "Under the hood, LanceDB is converting the input data into an Apache Arrow table and persisting it to disk in [Lance format](https://www.github.com/lancedb/lance)."
 
@@ -108,7 +116,7 @@ Once created, you can open a table using the following code:
 
 === "Javascript"
       ```javascript
-      const tbl = await db.openTable("my_table");
+      const tbl = await db.openTable("myTable");
       ```
 
       If you forget the name of your table, you can always get a listing of all table names:
@@ -194,10 +202,17 @@ Use the `drop_table()` method on the database to remove a table.
       db.drop_table("my_table")
       ```
 
-This permanently removes the table and is not recoverable, unlike deleting rows.
-By default, if the table does not exist an exception is raised. To suppress this,
-you can pass in `ignore_missing=True`.
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
 
+=== "JavaScript"
+      ```javascript
+      await db.dropTable('myTable')
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised. 
 
 ## What's next
 

docs/src/guides/tables.md

@@ -201,8 +201,8 @@ A Table is a collection of Records in a LanceDB Database. You can follow along o
     ```javascript
     data
     const tb = await db.createTable("my_table",
-                  data=[{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
-                        {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
+      [{"vector": [3.1, 4.1], "item": "foo", "price": 10.0},
+       {"vector": [5.9, 26.5], "item": "bar", "price": 20.0}])
     ```
 
     !!! info "Note"

docs/src/search.md

@@ -118,4 +118,101 @@ However, fast vector search using indices often entails making a trade-off with
 This is why it is often called **Approximate Nearest Neighbors (ANN)** search, while the Flat Search (KNN)
 always returns 100% recall.
 
-See [ANN Index](ann_indexes.md) for more details.
+See [ANN Index](ann_indexes.md) for more details.
+
+
+### Output formats
+
+LanceDB returns results in many different formats commonly used in python.
+Let's create a LanceDB table with a nested schema:
+
+```python
+from datetime import datetime
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+import numpy as np
+from pydantic import BaseModel
+uri = "data/sample-lancedb-nested"
+
+class Metadata(BaseModel):
+    source: str
+    timestamp: datetime
+
+class Document(BaseModel):
+    content: str
+    meta: Metadata
+
+class LanceSchema(LanceModel):
+    id: str
+    vector: Vector(1536)
+    payload: Document
+
+# Let's add 100 sample rows to our dataset
+data = [LanceSchema(
+    id=f"id{i}",
+    vector=np.random.randn(1536),
+    payload=Document(
+        content=f"document{i}", meta=Metadata(source=f"source{i%10}", timestamp=datetime.now())
+    ),
+) for i in range(100)]
+
+tbl = db.create_table("documents", data=data)
+```
+
+#### As a pyarrow table
+
+Using `to_arrow()` we can get the results back as a pyarrow Table.
+This result table has the same columns as the LanceDB table, with 
+the addition of an `_distance` column for vector search or a `score`
+column for full text search.
+
+```python
+tbl.search(np.random.randn(1536)).to_arrow()
+```
+
+#### As a pandas dataframe
+
+You can also get the results as a pandas dataframe.
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas()
+```
+
+While other formats like Arrow/Pydantic/Python dicts have a natural 
+way to handle nested schemas, pandas can only store nested data as a 
+python dict column, which makes it difficult to support nested references.
+So for convenience, you can also tell LanceDB to flatten a nested schema 
+when creating the pandas dataframe. 
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas(flatten=True)
+```
+
+If your table has a deeply nested struct, you can control how many levels
+of nesting to flatten by passing in a positive integer.
+
+```python
+tbl.search(np.random.randn(1536)).to_pandas(flatten=1)
+```
+
+
+#### As a list of python dicts
+
+You can of course return results as a list of python dicts.
+
+```python
+tbl.search(np.random.randn(1536)).to_list()
+```
+
+#### As a list of pydantic models
+
+We can add data using pydantic models, and we can certainly
+retrieve results as pydantic models
+
+```python
+tbl.search(np.random.randn(1536)).to_pydantic(LanceSchema)
+```
+
+Note that in this case the extra `_distance` field is discarded since
+it's not part of the LanceSchema.
+

docs/src/sql.md

@@ -22,7 +22,7 @@ import numpy as np
 uri = "data/sample-lancedb"
 db = lancedb.connect(uri)
 
-data = [{"vector": row, "item": f"item {i}"}
+data = [{"vector": row, "item": f"item {i}", "id": i}
      for i, row in enumerate(np.random.random((10_000, 2)).astype('int'))]
 
 tbl = db.create_table("my_vectors", data=data)
@@ -35,33 +35,25 @@ const db = await vectordb.connect('data/sample-lancedb')
 
 let data = []
 for (let i = 0; i < 10_000; i++) {
-     data.push({vector: Array(1536).fill(i), id: `${i}`, content: "", longId: `${i}`},)
+     data.push({vector: Array(1536).fill(i), id: i, item: `item ${i}`, strId: `${i}`})
 }
-const tbl = await db.createTable('my_vectors', data)
+const tbl = await db.createTable('myVectors', data)
 ```
 -->
 === "Python"
 
     ```python
     tbl.search([100, 102]) \
-       .where("""(
-        (label IN [10, 20])
-        AND
-        (note.email IS NOT NULL)
-        ) OR NOT note.created
-        """)
-
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)") \
+       .to_arrow()
     ```
+
 === "Javascript"
 
     ```javascript
-    tbl.search([100, 102])
-       .where(`(
-            (label IN [10, 20])
-            AND
-            (note.email IS NOT NULL)
-        ) OR NOT note.created
-       `)
+    await tbl.search(Array(1536).fill(0))
+       .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+       .execute()
     ```
 
 
@@ -118,3 +110,22 @@ The mapping from SQL types to Arrow types is:
 
 [^1]: See precision mapping in previous table.
 
+
+## Filtering without Vector Search
+
+You can also filter your data without search.
+
+=== "Python"
+      ```python
+      tbl.search().where("id=10").limit(10).to_arrow()
+      ```
+
+=== "JavaScript"
+      ```javascript
+      await tbl.where('id=10').limit(10).execute()
+      ```
+
+!!! warning
+    If your table is large, this could potentially return a very large
+    amount of data. Please be sure to use a `limit` clause unless
+    you're sure you want to return the whole result set.

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.3.9",
+  "version": "0.4.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.3.9",
+      "version": "0.4.0",
       "cpu": [
         "x64",
         "arm64"
@@ -53,11 +53,11 @@
         "uuid": "^9.0.0"
       },
       "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.3.9",
-        "@lancedb/vectordb-darwin-x64": "0.3.9",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.3.9",
-        "@lancedb/vectordb-linux-x64-gnu": "0.3.9",
-        "@lancedb/vectordb-win32-x64-msvc": "0.3.9"
+        "@lancedb/vectordb-darwin-arm64": "0.4.0",
+        "@lancedb/vectordb-darwin-x64": "0.4.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.4.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.4.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.4.0"
       }
     },
     "node_modules/@apache-arrow/ts": {
@@ -317,9 +317,9 @@
       }
     },
     "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.9.tgz",
-      "integrity": "sha512-irtAdfSRQDcfnMnB8T7D0atLFfu1MMZZ1JaxMKu24DDZ8e4IMYKUplxwvWni3241yA9yDE/pliRZCNQbQCEfrg==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.0.tgz",
+      "integrity": "sha512-cP6zGtBWXEcJHCI4uLNIP5ILtRvexvwmL8Uri1dnHG8dT8g12Ykug3BHO6Wt6wp/xASd2jJRIF/VAJsN9IeP1A==",
       "cpu": [
         "arm64"
       ],
@@ -329,9 +329,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.9.tgz",
-      "integrity": "sha512-4xXQoPheyIl1P5kRoKmZtaAHFrYdL9pw5yq+r6ewIx0TCemN4LSvzSUTqM5nZl3QPU8FeL0CGD8Gt2gMU0HQ2A==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.0.tgz",
+      "integrity": "sha512-ig0gV5ol1sFe2lb1HOatK0rizyj9I91WbnH79i7OdUl3nAQIcWm70CnxrPLtx0DS2NTGh2kFJbYCWcaUlu6YfA==",
       "cpu": [
         "x64"
       ],
@@ -341,9 +341,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.9.tgz",
-      "integrity": "sha512-WIxCZKnLeSlz0PGURtKSX6hJ4CYE2o5P+IFmmuWOWB1uNapQu6zOpea6rNxcRFHUA0IJdO02lVxVfn2hDX4SMg==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.0.tgz",
+      "integrity": "sha512-gMXIDT2kriAPDwWIRKXdaTCNdOeFGEok1S9Y30AOruHXddW1vCIo4JNJIYbBqHnwAeI4wI3ae6GRCFaf1UxO3g==",
       "cpu": [
         "arm64"
       ],
@@ -353,9 +353,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.9.tgz",
-      "integrity": "sha512-bQbcV9adKzYbJLNzDjk9OYsMnT2IjmieLfb4IQ1hj5IUoWfbg80Bd0+gZUnrmrhG6fe56TIriFZYQR9i7TSE9Q==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.0.tgz",
+      "integrity": "sha512-ZQ3lDrDSz1IKdx/mS9Lz08agFO+OD5oSFrrcFNCoT1+H93eS1mCLdmCoEARu3jKbx0tMs38l5J9yXZ2QmJye3w==",
       "cpu": [
         "x64"
       ],
@@ -365,9 +365,9 @@
       ]
     },
     "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.9.tgz",
-      "integrity": "sha512-7EXI7P1QvAfgJNPWWBMDOkoJ696gSBAClcyEJNYg0JV21jVFZRwJVI3bZXflesWduFi/mTuzPkFFA68us1u19A==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.0.tgz",
+      "integrity": "sha512-toNcNwBRE1sdsSf5hr7W8QiqZ33csc/knVEek4CyvYkZHJGh4Z6WI+DJUIASo5wzUez4TX7qUPpRPL9HuaPMCg==",
       "cpu": [
         "x64"
       ],
@@ -4869,33 +4869,33 @@
       }
     },
     "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.9.tgz",
-      "integrity": "sha512-irtAdfSRQDcfnMnB8T7D0atLFfu1MMZZ1JaxMKu24DDZ8e4IMYKUplxwvWni3241yA9yDE/pliRZCNQbQCEfrg==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.4.0.tgz",
+      "integrity": "sha512-cP6zGtBWXEcJHCI4uLNIP5ILtRvexvwmL8Uri1dnHG8dT8g12Ykug3BHO6Wt6wp/xASd2jJRIF/VAJsN9IeP1A==",
       "optional": true
     },
     "@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.9.tgz",
-      "integrity": "sha512-4xXQoPheyIl1P5kRoKmZtaAHFrYdL9pw5yq+r6ewIx0TCemN4LSvzSUTqM5nZl3QPU8FeL0CGD8Gt2gMU0HQ2A==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.4.0.tgz",
+      "integrity": "sha512-ig0gV5ol1sFe2lb1HOatK0rizyj9I91WbnH79i7OdUl3nAQIcWm70CnxrPLtx0DS2NTGh2kFJbYCWcaUlu6YfA==",
       "optional": true
     },
     "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.9.tgz",
-      "integrity": "sha512-WIxCZKnLeSlz0PGURtKSX6hJ4CYE2o5P+IFmmuWOWB1uNapQu6zOpea6rNxcRFHUA0IJdO02lVxVfn2hDX4SMg==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.4.0.tgz",
+      "integrity": "sha512-gMXIDT2kriAPDwWIRKXdaTCNdOeFGEok1S9Y30AOruHXddW1vCIo4JNJIYbBqHnwAeI4wI3ae6GRCFaf1UxO3g==",
       "optional": true
     },
     "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.9.tgz",
-      "integrity": "sha512-bQbcV9adKzYbJLNzDjk9OYsMnT2IjmieLfb4IQ1hj5IUoWfbg80Bd0+gZUnrmrhG6fe56TIriFZYQR9i7TSE9Q==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.4.0.tgz",
+      "integrity": "sha512-ZQ3lDrDSz1IKdx/mS9Lz08agFO+OD5oSFrrcFNCoT1+H93eS1mCLdmCoEARu3jKbx0tMs38l5J9yXZ2QmJye3w==",
       "optional": true
     },
     "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.9",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.9.tgz",
-      "integrity": "sha512-7EXI7P1QvAfgJNPWWBMDOkoJ696gSBAClcyEJNYg0JV21jVFZRwJVI3bZXflesWduFi/mTuzPkFFA68us1u19A==",
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.4.0.tgz",
+      "integrity": "sha512-toNcNwBRE1sdsSf5hr7W8QiqZ33csc/knVEek4CyvYkZHJGh4Z6WI+DJUIASo5wzUez4TX7qUPpRPL9HuaPMCg==",
       "optional": true
     },
     "@neon-rs/cli": {

node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vectordb",
-  "version": "0.3.9",
+  "version": "0.4.0",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -81,10 +81,10 @@
     }
   },
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.3.9",
-    "@lancedb/vectordb-darwin-x64": "0.3.9",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.3.9",
-    "@lancedb/vectordb-linux-x64-gnu": "0.3.9",
-    "@lancedb/vectordb-win32-x64-msvc": "0.3.9"
+    "@lancedb/vectordb-darwin-arm64": "0.4.0",
+    "@lancedb/vectordb-darwin-x64": "0.4.0",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.0",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.0",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.0"
   }
 }

node/src/index.ts

@@ -21,15 +21,7 @@ import type { EmbeddingFunction } from './embedding/embedding_function'
 import { RemoteConnection } from './remote'
 import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'
-import {
-  type Connection, type CreateTableOptions, type Table,
-  type VectorIndexParams, type UpdateArgs, type UpdateSqlArgs,
-  type VectorIndex, type IndexStats,
-  type ConnectionOptions, WriteMode, type WriteOptions
-} from './types'
-import { toSQL } from './util'
-
-export { type WriteMode }
+import { type Literal, toSQL } from './util'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
@@ -38,6 +30,30 @@ export { Query }
 export type { EmbeddingFunction }
 export { OpenAIEmbeddingFunction } from './embedding/openai'
 
+export interface AwsCredentials {
+  accessKeyId: string
+
+  secretKey: string
+
+  sessionToken?: string
+}
+
+export interface ConnectionOptions {
+  uri: string
+
+  awsCredentials?: AwsCredentials
+
+  awsRegion?: string
+
+  // API key for the remote connections
+  apiKey?: string
+  // Region to connect
+  region?: string
+
+  // override the host for the remote connections
+  hostOverride?: string
+}
+
 function getAwsArgs (opts: ConnectionOptions): any[] {
   const callArgs = []
   const awsCredentials = opts.awsCredentials
@@ -55,6 +71,23 @@ function getAwsArgs (opts: ConnectionOptions): any[] {
   return callArgs
 }
 
+export interface CreateTableOptions<T> {
+  // Name of Table
+  name: string
+
+  // Data to insert into the Table
+  data?: Array<Record<string, unknown>> | ArrowTable | undefined
+
+  // Optional Arrow Schema for this table
+  schema?: Schema | undefined
+
+  // Optional embedding function used to create embeddings
+  embeddingFunction?: EmbeddingFunction<T> | undefined
+
+  // WriteOptions for this operation
+  writeOptions?: WriteOptions | undefined
+}
+
 /**
  * Connect to a LanceDB instance at the given URI
  * @param uri The uri of the database.
@@ -83,6 +116,235 @@ export async function connect (arg: string | Partial<ConnectionOptions>): Promis
   return new LocalConnection(db, opts)
 }
 
+/**
+ * A LanceDB Connection that allows you to open tables and create new ones.
+ *
+ * Connection could be local against filesystem or remote against a server.
+ */
+export interface Connection {
+  uri: string
+
+  tableNames(): Promise<string[]>
+
+  /**
+   * Open a table in the database.
+   *
+   * @param name The name of the table.
+   * @param embeddings An embedding function to use on this table
+   */
+  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
+
+  /**
+   * Creates a new Table, optionally initializing it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Array of Records to be inserted into the table
+   * @param schema - An Arrow Schema that describe this table columns
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   * @param {WriteOptions} writeOptions - The write options to use when creating the table.
+   */
+  createTable<T> ({ name, data, schema, embeddingFunction, writeOptions }: CreateTableOptions<T>): Promise<Table<T>>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   */
+  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {WriteOptions} options - The write options to use when creating the table.
+   */
+  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
+  /**
+   * Creates a new Table and initialize it with new data.
+   *
+   * @param {string} name - The name of the table.
+   * @param data - Non-empty Array of Records to be inserted into the table
+   * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
+   * @param {WriteOptions} options - The write options to use when creating the table.
+   */
+  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
+
+  /**
+   * Drop an existing table.
+   * @param name The name of the table to drop.
+   */
+  dropTable(name: string): Promise<void>
+
+}
+
+/**
+ * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
+ */
+export interface Table<T = number[]> {
+  name: string
+
+  /**
+   * Creates a search query to find the nearest neighbors of the given search term
+   * @param query The query search term
+   */
+  search: (query: T) => Query<T>
+
+  /**
+   * Insert records into this Table.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  add: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Insert records into this Table, replacing its contents.
+   *
+   * @param data Records to be inserted into the Table
+   * @return The number of rows added to the table
+   */
+  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
+
+  /**
+   * Create an ANN index on this Table vector index.
+   *
+   * @param indexParams The parameters of this Index, @see VectorIndexParams.
+   */
+  createIndex: (indexParams: VectorIndexParams) => Promise<any>
+
+  /**
+   * Returns the number of rows in this table.
+   */
+  countRows: () => Promise<number>
+
+  /**
+   * Delete rows from this table.
+   *
+   * This can be used to delete a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param filter  A filter in the same format used by a sql WHERE clause. The
+   *                filter must not be empty.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [1, 2]},
+   *    {id: 2, vector: [3, 4]},
+   *    {id: 3, vector: [5, 6]},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   * await tbl.delete("id = 2")
+   * await tbl.countRows() // Returns 2
+   * ```
+   *
+   * If you have a list of values to delete, you can combine them into a
+   * stringified list and use the `IN` operator:
+   *
+   * ```ts
+   * const to_remove = [1, 5];
+   * await tbl.delete(`id IN (${to_remove.join(",")})`)
+   * await tbl.countRows() // Returns 1
+   * ```
+   */
+  delete: (filter: string) => Promise<void>
+
+  /**
+   * Update rows in this table.
+   *
+   * This can be used to update a single row, many rows, all rows, or
+   * sometimes no rows (if your predicate matches nothing).
+   *
+   * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect("./.lancedb")
+   * const data = [
+   *    {id: 1, vector: [3, 3], name: 'Ye'},
+   *    {id: 2, vector: [4, 4], name: 'Mike'},
+   * ];
+   * const tbl = await con.createTable("my_table", data)
+   *
+   * await tbl.update({
+   *   filter: "id = 2",
+   *   updates: { vector: [2, 2], name: "Michael" },
+   * })
+   *
+   * let results = await tbl.search([1, 1]).execute();
+   * // Returns [
+   * //   {id: 2, vector: [2, 2], name: 'Michael'}
+   * //   {id: 1, vector: [3, 3], name: 'Ye'}
+   * // ]
+   * ```
+   *
+   */
+  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
+
+  /**
+   * List the indicies on this table.
+   */
+  listIndices: () => Promise<VectorIndex[]>
+
+  /**
+   * Get statistics about an index.
+   */
+  indexStats: (indexUuid: string) => Promise<IndexStats>
+}
+
+export interface UpdateArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set
+   */
+  values: Record<string, Literal>
+}
+
+export interface UpdateSqlArgs {
+  /**
+   * A filter in the same format used by a sql WHERE clause. The filter may be empty,
+   * in which case all rows will be updated.
+   */
+  where?: string
+
+  /**
+   * A key-value map of updates. The keys are the column names, and the values are the
+   * new values to set as SQL expressions.
+   */
+  valuesSql: Record<string, string>
+}
+
+export interface VectorIndex {
+  columns: string[]
+  name: string
+  uuid: string
+}
+
+export interface IndexStats {
+  numIndexedRows: number | null
+  numUnindexedRows: number | null
+}
+
 /**
  * A connection to a LanceDB database.
  */
@@ -430,6 +692,88 @@ export interface CompactionMetrics {
   filesAdded: number
 }
 
+/// Config to build IVF_PQ index.
+///
+export interface IvfPQIndexConfig {
+  /**
+   * The column to be indexed
+   */
+  column?: string
+
+  /**
+   * A unique name for the index
+   */
+  index_name?: string
+
+  /**
+   * Metric type, L2 or Cosine
+   */
+  metric_type?: MetricType
+
+  /**
+   * The number of partitions this index
+   */
+  num_partitions?: number
+
+  /**
+   * The max number of iterations for kmeans training.
+   */
+  max_iters?: number
+
+  /**
+   * Train as optimized product quantization.
+   */
+  use_opq?: boolean
+
+  /**
+   * Number of subvectors to build PQ code
+   */
+  num_sub_vectors?: number
+  /**
+   * The number of bits to present one PQ centroid.
+   */
+  num_bits?: number
+
+  /**
+   * Max number of iterations to train OPQ, if `use_opq` is true.
+   */
+  max_opq_iters?: number
+
+  /**
+   * Replace an existing index with the same name if it exists.
+   */
+  replace?: boolean
+
+  /**
+   * Cache size of the index
+   */
+  index_cache_size?: number
+
+  type: 'ivf_pq'
+}
+
+export type VectorIndexParams = IvfPQIndexConfig
+
+/**
+ * Write mode for writing a table.
+ */
+export enum WriteMode {
+  /** Create a new {@link Table}. */
+  Create = 'create',
+  /** Overwrite the existing {@link Table} if presented. */
+  Overwrite = 'overwrite',
+  /** Append new data to the table. */
+  Append = 'append'
+}
+
+/**
+ * Write options when creating a Table.
+ */
+export interface WriteOptions {
+  /** A {@link WriteMode} to use on this operation */
+  writeMode?: WriteMode
+}
+
 export class DefaultWriteOptions implements WriteOptions {
   writeMode = WriteMode.Create
 }
@@ -438,3 +782,23 @@ export function isWriteOptions (value: any): value is WriteOptions {
   return Object.keys(value).length === 1 &&
       (value.writeMode === undefined || typeof value.writeMode === 'string')
 }
+
+/**
+ * Distance metrics type.
+ */
+export enum MetricType {
+  /**
+   * Euclidean distance
+   */
+  L2 = 'l2',
+
+  /**
+   * Cosine distance
+   */
+  Cosine = 'cosine',
+
+  /**
+   * Dot product
+   */
+  Dot = 'dot'
+}

node/src/integration_test/test.ts

@@ -0,0 +1,180 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe } from 'mocha'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+import { v4 as uuidv4 } from 'uuid'
+
+import * as lancedb from '../index'
+import { tmpdir } from 'os'
+import * as fs from 'fs'
+import * as path from 'path'
+
+const assert = chai.assert
+chai.use(chaiAsPromised)
+
+describe('LanceDB AWS Integration test', function () {
+  it('s3+ddb schema is processed correctly', async function () {
+    this.timeout(15000)
+
+    // WARNING: specifying engine is NOT a publicly supported feature in lancedb yet
+    // THE API WILL CHANGE
+    const conn = await lancedb.connect('s3://lancedb-integtest?engine=ddb&ddbTableName=lancedb-integtest')
+    const data = [{ vector: Array(128).fill(1.0) }]
+
+    const tableName = uuidv4()
+    let table = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
+
+    const futs = [table.add(data), table.add(data), table.add(data), table.add(data), table.add(data)]
+    await Promise.allSettled(futs)
+
+    table = await conn.openTable(tableName)
+    assert.equal(await table.countRows(), 6)
+  })
+})
+
+describe('LanceDB Mirrored Store Integration test', function () {
+  it('s3://...?mirroredStore=... param is processed correctly', async function () {
+    this.timeout(600000)
+
+    const dir = tmpdir()
+    console.log(dir)
+    const conn = await lancedb.connect(`s3://lancedb-integtest?mirroredStore=${dir}`)
+    const data = Array(200).fill({ vector: Array(128).fill(1.0), id: 0 })
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 1 }))
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 2 }))
+    data.push(...Array(200).fill({ vector: Array(128).fill(1.0), id: 3 }))
+
+    const tableName = uuidv4()
+
+    // try create table and check if it's mirrored
+    const t = await conn.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite })
+
+    const mirroredPath = path.join(dir, `${tableName}.lance`)
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be three dirs
+      assert.equal(files.length, 3)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_versions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.manifest'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+    })
+
+    // try create index and check if it's mirrored
+    await t.createIndex({ column: 'vector', type: 'ivf_pq' })
+
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be four dirs
+      assert.equal(files.length, 4)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+      assert.isTrue(files[2].isDirectory())
+
+      // Two TXs now
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 2)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+        assert.isTrue(files[1].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].isDirectory())
+
+        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
+          if (err != null) throw err
+
+          assert.equal(files.length, 1)
+          assert.isTrue(files[0].isFile())
+          assert.isTrue(files[0].name.endsWith('.idx'))
+        })
+      })
+    })
+
+    // try delete and check if it's mirrored
+    await t.delete('id = 0')
+
+    fs.readdir(mirroredPath, { withFileTypes: true }, (err, files) => {
+      if (err != null) throw err
+      // there should be five dirs
+      assert.equal(files.length, 5)
+      assert.isTrue(files[0].isDirectory())
+      assert.isTrue(files[1].isDirectory())
+      assert.isTrue(files[2].isDirectory())
+      assert.isTrue(files[3].isDirectory())
+      assert.isTrue(files[4].isDirectory())
+
+      // Three TXs now
+      fs.readdir(path.join(mirroredPath, '_transactions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 3)
+        assert.isTrue(files[0].name.endsWith('.txn'))
+        assert.isTrue(files[1].name.endsWith('.txn'))
+      })
+
+      fs.readdir(path.join(mirroredPath, 'data'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.lance'))
+      })
+
+      fs.readdir(path.join(mirroredPath, '_indices'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].isDirectory())
+
+        fs.readdir(path.join(mirroredPath, '_indices', files[0].name), { withFileTypes: true }, (err, files) => {
+          if (err != null) throw err
+
+          assert.equal(files.length, 1)
+          assert.isTrue(files[0].isFile())
+          assert.isTrue(files[0].name.endsWith('.idx'))
+        })
+      })
+
+      fs.readdir(path.join(mirroredPath, '_deletions'), { withFileTypes: true }, (err, files) => {
+        if (err != null) throw err
+        assert.equal(files.length, 1)
+        assert.isTrue(files[0].name.endsWith('.arrow'))
+      })
+    })
+  })
+})

node/src/query.ts

@@ -14,16 +14,10 @@
 
 import { Vector, tableFromIPC } from 'apache-arrow'
 import { type EmbeddingFunction } from './embedding/embedding_function'
-import { type MetricType } from './types'
+import { type MetricType } from '.'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-// const { tableSearch } = require('../native.js')
-
-const tableSearch = async function (args: any, arg2: any): Promise<any> {
-  return await new Promise((resolve, reject) => {
-    resolve('')
-  })
-}
+const { tableSearch } = require('../native.js')
 
 /**
  * A builder for nearest neighbor queries for LanceDB.

node/src/remote/index.ts

@@ -13,21 +13,19 @@
 // limitations under the License.
 
 import {
-  type Table, type VectorIndexParams,
-  type VectorIndex,
+  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
+  type ConnectionOptions, type CreateTableOptions, type VectorIndex,
+  type WriteOptions,
   type IndexStats,
-  type UpdateArgs, type UpdateSqlArgs,
-  type Connection,
-  type ConnectionOptions, type CreateTableOptions,
-  type WriteOptions
-} from '../types'
-import { type EmbeddingFunction } from '../embedding/embedding_function'
+  type UpdateArgs, type UpdateSqlArgs
+} from '../index'
 import { Query } from '../query'
 
 import { Vector, Table as ArrowTable } from 'apache-arrow'
 import { HttpLancedbClient } from './client'
 import { isEmbeddingFunction } from '../embedding/embedding_function'
 import { createEmptyTable, fromRecordsToStreamBuffer, fromTableToStreamBuffer } from '../arrow'
+import { toSQL } from '../util'
 
 /**
  * Remote connection.
@@ -59,8 +57,8 @@ export class RemoteConnection implements Connection {
     return 'db://' + this._client.uri
   }
 
-  async tableNames (): Promise<string[]> {
-    const response = await this._client.get('/v1/table/')
+  async tableNames (pageToken: string = '', limit: number = 10): Promise<string[]> {
+    const response = await this._client.get('/v1/table/', { limit, page_token: pageToken })
     return response.data.tables
   }
 
@@ -197,6 +195,17 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return this._name
   }
 
+  get schema (): Promise<any> {
+    return this._client.post(`/v1/table/${this._name}/describe/`).then(res => {
+      if (res.status !== 200) {
+        throw new Error(`Server Error, status: ${res.status}, ` +
+        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`)
+      }
+      return res.data?.schema
+    })
+  }
+
   search (query: T): Query<T> {
     return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
   }
@@ -237,8 +246,41 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return data.length
   }
 
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
-    throw new Error('Not implemented')
+  async createIndex (indexParams: VectorIndexParams): Promise<void> {
+    const unsupportedParams = [
+      'index_name',
+      'num_partitions',
+      'max_iters',
+      'use_opq',
+      'num_sub_vectors',
+      'num_bits',
+      'max_opq_iters',
+      'replace'
+    ]
+    for (const param of unsupportedParams) {
+      // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+      if (indexParams[param as keyof VectorIndexParams]) {
+        throw new Error(`${param} is not supported for remote connections`)
+      }
+    }
+
+    const column = indexParams.column ?? 'vector'
+    const indexType = 'vector' // only vector index is supported for remote connections
+    const metricType = indexParams.metric_type ?? 'L2'
+    const indexCacheSize = indexParams ?? null
+
+    const data = {
+      column,
+      index_type: indexType,
+      metric_type: metricType,
+      index_cache_size: indexCacheSize
+    }
+    const res = await this._client.post(`/v1/table/${this._name}/create_index/`, data)
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
   }
 
   async countRows (): Promise<number> {
@@ -251,7 +293,23 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   async update (args: UpdateArgs | UpdateSqlArgs): Promise<void> {
-    throw new Error('Not implemented')
+    let filter: string | null
+    let updates: Record<string, string>
+
+    if ('valuesSql' in args) {
+      filter = args.where ?? null
+      updates = args.valuesSql
+    } else {
+      filter = args.where ?? null
+      updates = {}
+      for (const [key, value] of Object.entries(args.values)) {
+        updates[key] = toSQL(value)
+      }
+    }
+    await this._client.post(`/v1/table/${this._name}/update/`, {
+      predicate: filter,
+      updates: Object.entries(updates).map(([key, value]) => [key, value])
+    })
   }
 
   async listIndices (): Promise<VectorIndex[]> {

node/src/test/embedding/openai.ts

@@ -0,0 +1,57 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe } from 'mocha'
+import { assert } from 'chai'
+
+import { OpenAIEmbeddingFunction } from '../../embedding/openai'
+import { isEmbeddingFunction } from '../../embedding/embedding_function'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { OpenAIApi } = require('openai')
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { stub } = require('sinon')
+
+describe('OpenAPIEmbeddings', function () {
+  const stubValue = {
+    data: {
+      data: [
+        {
+          embedding: Array(1536).fill(1.0)
+        },
+        {
+          embedding: Array(1536).fill(2.0)
+        }
+      ]
+    }
+  }
+
+  describe('#embed', function () {
+    it('should create vector embeddings', async function () {
+      const openAIStub = stub(OpenAIApi.prototype, 'createEmbedding').returns(stubValue)
+      const f = new OpenAIEmbeddingFunction('text', 'sk-key')
+      const vectors = await f.embed(['abc', 'def'])
+      assert.isTrue(openAIStub.calledOnce)
+      assert.equal(vectors.length, 2)
+      assert.deepEqual(vectors[0], stubValue.data.data[0].embedding)
+      assert.deepEqual(vectors[1], stubValue.data.data[1].embedding)
+    })
+  })
+
+  describe('isEmbeddingFunction', function () {
+    it('should match the isEmbeddingFunction guard', function () {
+      assert.isTrue(isEmbeddingFunction(new OpenAIEmbeddingFunction('text', 'sk-key')))
+    })
+  })
+})

node/src/test/io.ts

@@ -0,0 +1,76 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// IO tests
+
+import { describe } from 'mocha'
+import { assert } from 'chai'
+
+import * as lancedb from '../index'
+import { type ConnectionOptions } from '../index'
+
+describe('LanceDB S3 client', function () {
+  if (process.env.TEST_S3_BASE_URL != null) {
+    const baseUri = process.env.TEST_S3_BASE_URL
+    it('should have a valid url', async function () {
+      const opts = { uri: `${baseUri}/valid_url` }
+      const table = await createTestDB(opts, 2, 20)
+      const con = await lancedb.connect(opts)
+      assert.equal(con.uri, opts.uri)
+
+      const results = await table.search([0.1, 0.3]).limit(5).execute()
+      assert.equal(results.length, 5)
+    }).timeout(10_000)
+  } else {
+    describe.skip('Skip S3 test', function () {})
+  }
+
+  if (process.env.TEST_S3_BASE_URL != null && process.env.TEST_AWS_ACCESS_KEY_ID != null && process.env.TEST_AWS_SECRET_ACCESS_KEY != null) {
+    const baseUri = process.env.TEST_S3_BASE_URL
+    it('use custom credentials', async function () {
+      const opts: ConnectionOptions = {
+        uri: `${baseUri}/custom_credentials`,
+        awsCredentials: {
+          accessKeyId: process.env.TEST_AWS_ACCESS_KEY_ID as string,
+          secretKey: process.env.TEST_AWS_SECRET_ACCESS_KEY as string
+        }
+      }
+      const table = await createTestDB(opts, 2, 20)
+      console.log(table)
+      const con = await lancedb.connect(opts)
+      console.log(con)
+      assert.equal(con.uri, opts.uri)
+
+      const results = await table.search([0.1, 0.3]).limit(5).execute()
+      assert.equal(results.length, 5)
+    }).timeout(10_000)
+  } else {
+    describe.skip('Skip S3 test', function () {})
+  }
+})
+
+async function createTestDB (opts: ConnectionOptions, numDimensions: number = 2, numRows: number = 2): Promise<lancedb.Table> {
+  const con = await lancedb.connect(opts)
+
+  const data = []
+  for (let i = 0; i < numRows; i++) {
+    const vector = []
+    for (let j = 0; j < numDimensions; j++) {
+      vector.push(i + (j * 0.1))
+    }
+    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
+  }
+
+  return await con.createTable('vectors_2', data)
+}

node/src/test/test.ts

@@ -0,0 +1,616 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { describe } from 'mocha'
+import { track } from 'temp'
+import * as chai from 'chai'
+import * as chaiAsPromised from 'chai-as-promised'
+
+import * as lancedb from '../index'
+import { type AwsCredentials, type EmbeddingFunction, MetricType, Query, WriteMode, DefaultWriteOptions, isWriteOptions, type LocalTable } from '../index'
+import { FixedSizeList, Field, Int32, makeVector, Schema, Utf8, Table as ArrowTable, vectorFromArray, Float32 } from 'apache-arrow'
+
+const expect = chai.expect
+const assert = chai.assert
+chai.use(chaiAsPromised)
+
+describe('LanceDB client', function () {
+  describe('when creating a connection to lancedb', function () {
+    it('should have a valid url', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      assert.equal(con.uri, uri)
+    })
+
+    it('should accept an options object', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect({ uri })
+      assert.equal(con.uri, uri)
+    })
+
+    it('should accept custom aws credentials', async function () {
+      const uri = await createTestDB()
+      const awsCredentials: AwsCredentials = {
+        accessKeyId: '',
+        secretKey: ''
+      }
+      const con = await lancedb.connect({ uri, awsCredentials })
+      assert.equal(con.uri, uri)
+    })
+
+    it('should return the existing table names', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+  })
+
+  describe('when querying an existing dataset', function () {
+    it('should open a table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      assert.equal(table.name, 'vectors')
+    })
+
+    it('execute a query', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const results = await table.search([0.1, 0.3]).execute()
+
+      assert.equal(results.length, 2)
+      assert.equal(results[0].price, 10)
+      const vector = results[0].vector as Float32Array
+      assert.approximately(vector[0], 0.0, 0.2)
+      assert.approximately(vector[0], 0.1, 0.3)
+    })
+
+    it('limits # of results', async function () {
+      const uri = await createTestDB(2, 100)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      let results = await table.search([0.1, 0.3]).limit(1).execute()
+      assert.equal(results.length, 1)
+      assert.equal(results[0].id, 1)
+
+      // there is a default limit if unspecified
+      results = await table.search([0.1, 0.3]).execute()
+      assert.equal(results.length, 10)
+    })
+
+    it('uses a filter / where clause without vector search', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 50)
+      }
+
+      const uri = await createTestDB(2, 100)
+      const con = await lancedb.connect(uri)
+      const table = (await con.openTable('vectors')) as LocalTable
+      let results = await table.filter('id % 2 = 0').execute()
+      assertResults(results)
+      results = await table.where('id % 2 = 0').execute()
+      assertResults(results)
+    })
+
+    it('uses a filter / where clause', async function () {
+      // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+      const assertResults = (results: Array<Record<string, unknown>>) => {
+        assert.equal(results.length, 1)
+        assert.equal(results[0].id, 2)
+      }
+
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      let results = await table.search([0.1, 0.1]).filter('id == 2').execute()
+      assertResults(results)
+      results = await table.search([0.1, 0.1]).where('id == 2').execute()
+      assertResults(results)
+    })
+
+    it('should correctly process prefilter/postfilter', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      // post filter should return less than the limit
+      let results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(false).execute()
+      assert.isTrue(results.length < 10)
+
+      // pre filter should return exactly the limit
+      results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
+    it('select only a subset of columns', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const results = await table.search([0.1, 0.1]).select(['is_active']).execute()
+      assert.equal(results.length, 2)
+      // vector and _distance are always returned
+      assert.isDefined(results[0].vector)
+      assert.isDefined(results[0]._distance)
+      assert.isDefined(results[0].is_active)
+
+      assert.isUndefined(results[0].id)
+      assert.isUndefined(results[0].name)
+      assert.isUndefined(results[0].price)
+    })
+  })
+
+  describe('when creating a new dataset', function () {
+    it('create an empty table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('id', new Int32()), new Field('name', new Utf8())]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      assert.equal(table.name, 'vectors')
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
+    it('create a table with a empty data array', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('id', new Int32()), new Field('name', new Utf8())]
+      )
+      const table = await con.createTable({ name: 'vectors', schema, data: [] })
+      assert.equal(table.name, 'vectors')
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
+    it('create a table from an Arrow Table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const i32s = new Int32Array(new Array<number>(10))
+      const i32 = makeVector(i32s)
+
+      const data = new ArrowTable({ vector: i32 })
+
+      const table = await con.createTable({ name: 'vectors', data })
+      assert.equal(table.name, 'vectors')
+      assert.equal(await table.countRows(), 10)
+      assert.deepEqual(await con.tableNames(), ['vectors'])
+    })
+
+    it('creates a new table from javascript objects', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      const tableName = `vectors_${Math.floor(Math.random() * 100)}`
+      const table = await con.createTable(tableName, data)
+      assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 2)
+    })
+
+    it('fails to create a new table when the vector column is missing', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, price: 10 }
+      ]
+
+      const create = con.createTable('missing_vector', data)
+      await expect(create).to.be.rejectedWith(Error, 'column \'vector\' is missing')
+    })
+
+    it('use overwrite flag to overwrite existing table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      const tableName = 'overwrite'
+      await con.createTable(tableName, data, { writeMode: WriteMode.Create })
+
+      const newData = [
+        { id: 1, vector: [0.1, 0.2], price: 10 },
+        { id: 2, vector: [1.1, 1.2], price: 50 },
+        { id: 3, vector: [1.1, 1.2], price: 50 }
+      ]
+
+      await expect(con.createTable(tableName, newData)).to.be.rejectedWith(Error, 'already exists')
+
+      const table = await con.createTable(tableName, newData, { writeMode: WriteMode.Overwrite })
+      assert.equal(table.name, tableName)
+      assert.equal(await table.countRows(), 3)
+    })
+
+    it('appends records to an existing table ', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const data = [
+        { id: 1, vector: [0.1, 0.2], price: 10, name: 'a' },
+        { id: 2, vector: [1.1, 1.2], price: 50, name: 'b' }
+      ]
+
+      const table = await con.createTable('vectors', data)
+      assert.equal(await table.countRows(), 2)
+
+      const dataAdd = [
+        { id: 3, vector: [2.1, 2.2], price: 10, name: 'c' },
+        { id: 4, vector: [3.1, 3.2], price: 50, name: 'd' }
+      ]
+      await table.add(dataAdd)
+      assert.equal(await table.countRows(), 4)
+    })
+
+    it('overwrite all records in a table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      const dataOver = [
+        { vector: [2.1, 2.2], price: 10, name: 'foo' },
+        { vector: [3.1, 3.2], price: 50, name: 'bar' }
+      ]
+      await table.overwrite(dataOver)
+      assert.equal(await table.countRows(), 2)
+    })
+
+    it('can update records in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update the records using a literal value', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ where: 'price = 10', values: { price: 100 } })
+      const results = await table.search([0.1, 0.2]).execute()
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 11)
+    })
+
+    it('can update every record in the table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.update({ valuesSql: { price: '100' } })
+      const results = await table.search([0.1, 0.2]).execute()
+
+      assert.equal(results[0].price, 100)
+      assert.equal(results[1].price, 100)
+    })
+
+    it('can delete records from a table', async function () {
+      const uri = await createTestDB()
+      const con = await lancedb.connect(uri)
+
+      const table = await con.openTable('vectors')
+      assert.equal(await table.countRows(), 2)
+
+      await table.delete('price = 10')
+      assert.equal(await table.countRows(), 1)
+    })
+  })
+
+  describe('when searching an empty dataset', function () {
+    it('should not fail', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      const result = await table.search(Array(128).fill(0.1)).execute()
+      assert.isEmpty(result)
+    })
+  })
+
+  describe('when searching an empty-after-delete dataset', function () {
+    it('should not fail', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+
+      const schema = new Schema(
+        [new Field('vector', new FixedSizeList(128, new Field('float32', new Float32())))]
+      )
+      const table = await con.createTable({ name: 'vectors', schema })
+      await table.add([{ vector: Array(128).fill(0.1) }])
+      // https://github.com/lancedb/lance/issues/1635
+      await table.delete('true')
+      const result = await table.search(Array(128).fill(0.1)).execute()
+      assert.isEmpty(result)
+    })
+  })
+
+  describe('when creating a vector index', function () {
+    it('overwrite all records in a table', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+    }).timeout(10_000) // Timeout is high partially because GH macos runner is pretty slow
+
+    it('replace an existing index', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      // Replace should fail if the index already exists
+      await expect(table.createIndex({
+        type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2, replace: false
+      })
+      ).to.be.rejectedWith('LanceError(Index)')
+
+      // Default replace = true
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+    }).timeout(50_000)
+
+    it('it should fail when the column is not a vector', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+      await expect(createIndex).to.be.rejectedWith(/VectorIndex requires the column data type to be fixed size list of float32s/)
+    })
+
+    it('it should fail when the column is not a vector', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
+      await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
+    })
+
+    it('should be able to list index and stats', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      const indices = await table.listIndices()
+      expect(indices).to.have.lengthOf(1)
+      expect(indices[0].name).to.equal('vector_idx')
+      expect(indices[0].uuid).to.not.be.equal(undefined)
+      expect(indices[0].columns).to.have.lengthOf(1)
+      expect(indices[0].columns[0]).to.equal('vector')
+
+      const stats = await table.indexStats(indices[0].uuid)
+      expect(stats.numIndexedRows).to.equal(300)
+      expect(stats.numUnindexedRows).to.equal(0)
+    }).timeout(50_000)
+  })
+
+  describe('when using a custom embedding function', function () {
+    class TextEmbedding implements EmbeddingFunction<string> {
+      sourceColumn: string
+
+      constructor (targetColumn: string) {
+        this.sourceColumn = targetColumn
+      }
+
+      _embedding_map = new Map<string, number[]>([
+        ['foo', [2.1, 2.2]],
+        ['bar', [3.1, 3.2]]
+      ])
+
+      async embed (data: string[]): Promise<number[][]> {
+        return data.map(datum => this._embedding_map.get(datum) ?? [0.0, 0.0])
+      }
+    }
+
+    it('should encode the original data into embeddings', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+      const embeddings = new TextEmbedding('name')
+
+      const data = [
+        { price: 10, name: 'foo' },
+        { price: 50, name: 'bar' }
+      ]
+      const table = await con.createTable('vectors', data, embeddings, { writeMode: WriteMode.Create })
+      const results = await table.search('foo').execute()
+      assert.equal(results.length, 2)
+    })
+
+    it('should create embeddings for Arrow Table', async function () {
+      const dir = await track().mkdir('lancejs')
+      const con = await lancedb.connect(dir)
+      const embeddingFunction = new TextEmbedding('name')
+
+      const names = vectorFromArray(['foo', 'bar'], new Utf8())
+      const data = new ArrowTable({ name: names })
+
+      const table = await con.createTable({ name: 'vectors', data, embeddingFunction })
+      assert.equal(table.name, 'vectors')
+      const results = await table.search('foo').execute()
+      assert.equal(results.length, 2)
+    })
+  })
+})
+
+describe('Remote LanceDB client', function () {
+  describe('when the server is not reachable', function () {
+    it('produces a network error', async function () {
+      const con = await lancedb.connect({
+        uri: 'db://test-1234',
+        region: 'asdfasfasfdf',
+        apiKey: 'some-api-key'
+      })
+
+      // GET
+      try {
+        await con.tableNames()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // POST
+      try {
+        await con.createTable({ name: 'vectors', schema: new Schema([]) })
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // Search
+      const table = await con.openTable('vectors')
+      try {
+        await table.search([0.1, 0.3]).execute()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+    })
+  })
+})
+
+describe('Query object', function () {
+  it('sets custom parameters', async function () {
+    const query = new Query([0.1, 0.3])
+      .limit(1)
+      .metricType(MetricType.Cosine)
+      .refineFactor(100)
+      .select(['a', 'b'])
+      .nprobes(20) as Record<string, any>
+    assert.equal(query._limit, 1)
+    assert.equal(query._metricType, MetricType.Cosine)
+    assert.equal(query._refineFactor, 100)
+    assert.equal(query._nprobes, 20)
+    assert.deepEqual(query._select, ['a', 'b'])
+  })
+})
+
+async function createTestDB (numDimensions: number = 2, numRows: number = 2): Promise<string> {
+  const dir = await track().mkdir('lancejs')
+  const con = await lancedb.connect(dir)
+
+  const data = []
+  for (let i = 0; i < numRows; i++) {
+    const vector = []
+    for (let j = 0; j < numDimensions; j++) {
+      vector.push(i + (j * 0.1))
+    }
+    data.push({ id: i + 1, name: `name_${i}`, price: i + 10, is_active: (i % 2 === 0), vector })
+  }
+
+  await con.createTable('vectors', data)
+  return dir
+}
+
+describe('Drop table', function () {
+  it('drop a table', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+
+    const data = [
+      { price: 10, name: 'foo', vector: [1, 2, 3] },
+      { price: 50, name: 'bar', vector: [4, 5, 6] }
+    ]
+    await con.createTable('t1', data)
+    await con.createTable('t2', data)
+
+    assert.deepEqual(await con.tableNames(), ['t1', 't2'])
+
+    await con.dropTable('t1')
+    assert.deepEqual(await con.tableNames(), ['t2'])
+  })
+})
+
+describe('WriteOptions', function () {
+  context('#isWriteOptions', function () {
+    it('should not match empty object', function () {
+      assert.equal(isWriteOptions({}), false)
+    })
+    it('should match write options', function () {
+      assert.equal(isWriteOptions({ writeMode: WriteMode.Create }), true)
+    })
+    it('should match undefined write mode', function () {
+      assert.equal(isWriteOptions({ writeMode: undefined }), true)
+    })
+    it('should match default write options', function () {
+      assert.equal(isWriteOptions(new DefaultWriteOptions()), true)
+    })
+  })
+})
+
+describe('Compact and cleanup', function () {
+  it('can cleanup after compaction', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+
+    const data = [
+      { price: 10, name: 'foo', vector: [1, 2, 3] },
+      { price: 50, name: 'bar', vector: [4, 5, 6] }
+    ]
+    const table = await con.createTable('t1', data) as LocalTable
+
+    const newData = [
+      { price: 30, name: 'baz', vector: [7, 8, 9] }
+    ]
+    await table.add(newData)
+
+    const compactionMetrics = await table.compactFiles({
+      numThreads: 2
+    })
+    assert.equal(compactionMetrics.fragmentsRemoved, 2)
+    assert.equal(compactionMetrics.fragmentsAdded, 1)
+    assert.equal(await table.countRows(), 3)
+
+    await table.cleanupOldVersions()
+    assert.equal(await table.countRows(), 3)
+
+    // should have no effect, but this validates the arguments are parsed.
+    await table.compactFiles({
+      targetRowsPerFragment: 102410,
+      maxRowsPerGroup: 1024,
+      materializeDeletions: true,
+      materializeDeletionsThreshold: 0.5,
+      numThreads: 2
+    })
+
+    const cleanupMetrics = await table.cleanupOldVersions(0, true)
+    assert.isAtLeast(cleanupMetrics.bytesRemoved, 1)
+    assert.isAtLeast(cleanupMetrics.oldVersions, 1)
+    assert.equal(await table.countRows(), 3)
+  })
+})

node/src/test/util.ts

@@ -0,0 +1,45 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { toSQL } from '../util'
+import * as chai from 'chai'
+
+const expect = chai.expect
+
+describe('toSQL', function () {
+  it('should turn string to SQL expression', function () {
+    expect(toSQL('foo')).to.equal("'foo'")
+  })
+
+  it('should turn number to SQL expression', function () {
+    expect(toSQL(123)).to.equal('123')
+  })
+
+  it('should turn boolean to SQL expression', function () {
+    expect(toSQL(true)).to.equal('TRUE')
+  })
+
+  it('should turn null to SQL expression', function () {
+    expect(toSQL(null)).to.equal('NULL')
+  })
+
+  it('should turn Date to SQL expression', function () {
+    const date = new Date('05 October 2011 14:48 UTC')
+    expect(toSQL(date)).to.equal("'2011-10-05T14:48:00.000Z'")
+  })
+
+  it('should turn array to SQL expression', function () {
+    expect(toSQL(['foo', 'bar', true, 1])).to.equal("['foo', 'bar', TRUE, 1]")
+  })
+})

node/src/types.ts

@@ -1,375 +0,0 @@
-
-import {
-  type Schema,
-  type Table as ArrowTable
-} from 'apache-arrow'
-
-import { type Literal } from './util'
-import type { EmbeddingFunction } from './embedding/embedding_function'
-import { type Query } from './query'
-
-export interface AwsCredentials {
-  accessKeyId: string
-
-  secretKey: string
-
-  sessionToken?: string
-}
-
-/**
- * Write options when creating a Table.
- */
-export interface WriteOptions {
-  /** A {@link WriteMode} to use on this operation */
-  writeMode?: WriteMode
-}
-
-/**
- * Write mode for writing a table.
- */
-export enum WriteMode {
-  /** Create a new {@link Table}. */
-  Create = 'create',
-  /** Overwrite the existing {@link Table} if presented. */
-  Overwrite = 'overwrite',
-  /** Append new data to the table. */
-  Append = 'append'
-}
-
-/**
- * A LanceDB Connection that allows you to open tables and create new ones.
- *
- * Connection could be local against filesystem or remote against a server.
- */
-export interface Connection {
-  uri: string
-
-  tableNames(): Promise<string[]>
-
-  /**
-    * Open a table in the database.
-    *
-    * @param name The name of the table.
-    * @param embeddings An embedding function to use on this table
-    */
-  openTable<T>(name: string, embeddings?: EmbeddingFunction<T>): Promise<Table<T>>
-
-  /**
-    * Creates a new Table, optionally initializing it with new data.
-    *
-    * @param {string} name - The name of the table.
-    * @param data - Array of Records to be inserted into the table
-    * @param schema - An Arrow Schema that describe this table columns
-    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-    * @param {WriteOptions} writeOptions - The write options to use when creating the table.
-    */
-  createTable<T> ({ name, data, schema, embeddingFunction, writeOptions }: CreateTableOptions<T>): Promise<Table<T>>
-
-  /**
-    * Creates a new Table and initialize it with new data.
-    *
-    * @param {string} name - The name of the table.
-    * @param data - Non-empty Array of Records to be inserted into the table
-    */
-  createTable (name: string, data: Array<Record<string, unknown>>): Promise<Table>
-
-  /**
-    * Creates a new Table and initialize it with new data.
-    *
-    * @param {string} name - The name of the table.
-    * @param data - Non-empty Array of Records to be inserted into the table
-    * @param {WriteOptions} options - The write options to use when creating the table.
-    */
-  createTable (name: string, data: Array<Record<string, unknown>>, options: WriteOptions): Promise<Table>
-
-  /**
-    * Creates a new Table and initialize it with new data.
-    *
-    * @param {string} name - The name of the table.
-    * @param data - Non-empty Array of Records to be inserted into the table
-    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-    */
-  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>): Promise<Table<T>>
-  /**
-    * Creates a new Table and initialize it with new data.
-    *
-    * @param {string} name - The name of the table.
-    * @param data - Non-empty Array of Records to be inserted into the table
-    * @param {EmbeddingFunction} embeddings - An embedding function to use on this table
-    * @param {WriteOptions} options - The write options to use when creating the table.
-    */
-  createTable<T> (name: string, data: Array<Record<string, unknown>>, embeddings: EmbeddingFunction<T>, options: WriteOptions): Promise<Table<T>>
-
-  /**
-    * Drop an existing table.
-    * @param name The name of the table to drop.
-    */
-  dropTable(name: string): Promise<void>
-
-}
-
-export interface CreateTableOptions<T> {
-  // Name of Table
-  name: string
-
-  // Data to insert into the Table
-  data?: Array<Record<string, unknown>> | ArrowTable | undefined
-
-  // Optional Arrow Schema for this table
-  schema?: Schema | undefined
-
-  // Optional embedding function used to create embeddings
-  embeddingFunction?: EmbeddingFunction<T> | undefined
-
-  // WriteOptions for this operation
-  writeOptions?: WriteOptions | undefined
-}
-
-export interface ConnectionOptions {
-  uri: string
-
-  awsCredentials?: AwsCredentials
-
-  awsRegion?: string
-
-  // API key for the remote connections
-  apiKey?: string
-  // Region to connect
-  region?: string
-
-  // override the host for the remote connections
-  hostOverride?: string
-}
-
-/**
- * Distance metrics type.
- */
-export enum MetricType {
-  /**
-    * Euclidean distance
-    */
-  L2 = 'l2',
-
-  /**
-    * Cosine distance
-    */
-  Cosine = 'cosine',
-
-  /**
-    * Dot product
-    */
-  Dot = 'dot'
-}
-
-/// Config to build IVF_PQ index.
-///
-export interface IvfPQIndexConfig {
-  /**
-    * The column to be indexed
-    */
-  column?: string
-
-  /**
-    * A unique name for the index
-    */
-  index_name?: string
-
-  /**
-    * Metric type, L2 or Cosine
-    */
-  metric_type?: MetricType
-
-  /**
-    * The number of partitions this index
-    */
-  num_partitions?: number
-
-  /**
-    * The max number of iterations for kmeans training.
-    */
-  max_iters?: number
-
-  /**
-    * Train as optimized product quantization.
-    */
-  use_opq?: boolean
-
-  /**
-    * Number of subvectors to build PQ code
-    */
-  num_sub_vectors?: number
-  /**
-    * The number of bits to present one PQ centroid.
-    */
-  num_bits?: number
-
-  /**
-    * Max number of iterations to train OPQ, if `use_opq` is true.
-    */
-  max_opq_iters?: number
-
-  /**
-    * Replace an existing index with the same name if it exists.
-    */
-  replace?: boolean
-
-  type: 'ivf_pq'
-}
-
-export type VectorIndexParams = IvfPQIndexConfig
-
-/**
- * A LanceDB Table is the collection of Records. Each Record has one or more vector fields.
- */
-export interface Table<T = number[]> {
-  name: string
-
-  /**
-     * Creates a search query to find the nearest neighbors of the given search term
-     * @param query The query search term
-     */
-  search: (query: T) => Query<T>
-
-  /**
-     * Insert records into this Table.
-     *
-     * @param data Records to be inserted into the Table
-     * @return The number of rows added to the table
-     */
-  add: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-     * Insert records into this Table, replacing its contents.
-     *
-     * @param data Records to be inserted into the Table
-     * @return The number of rows added to the table
-     */
-  overwrite: (data: Array<Record<string, unknown>>) => Promise<number>
-
-  /**
-     * Create an ANN index on this Table vector index.
-     *
-     * @param indexParams The parameters of this Index, @see VectorIndexParams.
-     */
-  createIndex: (indexParams: VectorIndexParams) => Promise<any>
-
-  /**
-     * Returns the number of rows in this table.
-     */
-  countRows: () => Promise<number>
-
-  /**
-     * Delete rows from this table.
-     *
-     * This can be used to delete a single row, many rows, all rows, or
-     * sometimes no rows (if your predicate matches nothing).
-     *
-     * @param filter  A filter in the same format used by a sql WHERE clause. The
-     *                filter must not be empty.
-     *
-     * @examples
-     *
-     * ```ts
-     * const con = await lancedb.connect("./.lancedb")
-     * const data = [
-     *    {id: 1, vector: [1, 2]},
-     *    {id: 2, vector: [3, 4]},
-     *    {id: 3, vector: [5, 6]},
-     * ];
-     * const tbl = await con.createTable("my_table", data)
-     * await tbl.delete("id = 2")
-     * await tbl.countRows() // Returns 2
-     * ```
-     *
-     * If you have a list of values to delete, you can combine them into a
-     * stringified list and use the `IN` operator:
-     *
-     * ```ts
-     * const to_remove = [1, 5];
-     * await tbl.delete(`id IN (${to_remove.join(",")})`)
-     * await tbl.countRows() // Returns 1
-     * ```
-     */
-  delete: (filter: string) => Promise<void>
-
-  /**
-     * Update rows in this table.
-     *
-     * This can be used to update a single row, many rows, all rows, or
-     * sometimes no rows (if your predicate matches nothing).
-     *
-     * @param args see {@link UpdateArgs} and {@link UpdateSqlArgs} for more details
-     *
-     * @examples
-     *
-     * ```ts
-     * const con = await lancedb.connect("./.lancedb")
-     * const data = [
-     *    {id: 1, vector: [3, 3], name: 'Ye'},
-     *    {id: 2, vector: [4, 4], name: 'Mike'},
-     * ];
-     * const tbl = await con.createTable("my_table", data)
-     *
-     * await tbl.update({
-     *   filter: "id = 2",
-     *   updates: { vector: [2, 2], name: "Michael" },
-     * })
-     *
-     * let results = await tbl.search([1, 1]).execute();
-     * // Returns [
-     * //   {id: 2, vector: [2, 2], name: 'Michael'}
-     * //   {id: 1, vector: [3, 3], name: 'Ye'}
-     * // ]
-     * ```
-     *
-     */
-  update: (args: UpdateArgs | UpdateSqlArgs) => Promise<void>
-
-  /**
-     * List the indicies on this table.
-     */
-  listIndices: () => Promise<VectorIndex[]>
-
-  /**
-     * Get statistics about an index.
-     */
-  indexStats: (indexUuid: string) => Promise<IndexStats>
-}
-export interface UpdateArgs {
-  /**
-     * A filter in the same format used by a sql WHERE clause. The filter may be empty,
-     * in which case all rows will be updated.
-     */
-  where?: string
-
-  /**
-     * A key-value map of updates. The keys are the column names, and the values are the
-     * new values to set
-     */
-  values: Record<string, Literal>
-}
-
-export interface UpdateSqlArgs {
-  /**
-     * A filter in the same format used by a sql WHERE clause. The filter may be empty,
-     * in which case all rows will be updated.
-     */
-  where?: string
-
-  /**
-     * A key-value map of updates. The keys are the column names, and the values are the
-     * new values to set as SQL expressions.
-     */
-  valuesSql: Record<string, string>
-}
-
-export interface VectorIndex {
-  columns: string[]
-  name: string
-  uuid: string
-}
-
-export interface IndexStats {
-  numIndexedRows: number | null
-  numUnindexedRows: number | null
-}

python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.4
+current_version = 0.4.0
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True

python/lancedb/db.py

@@ -23,7 +23,7 @@ from overrides import EnforceOverrides, override
 from pyarrow import fs
 
 from .table import LanceTable, Table
-from .util import fs_from_uri, get_uri_location, get_uri_scheme
+from .util import fs_from_uri, get_uri_location, get_uri_scheme, join_uri
 
 if TYPE_CHECKING:
     from .common import DATA, URI
@@ -288,14 +288,13 @@ class LanceDBConnection(DBConnection):
             A list of table names.
         """
         try:
-            filesystem, path = fs_from_uri(self.uri)
+            filesystem = fs_from_uri(self.uri)[0]
         except pa.ArrowInvalid:
             raise NotImplementedError("Unsupported scheme: " + self.uri)
 
         try:
-            paths = filesystem.get_file_info(
-                fs.FileSelector(get_uri_location(self.uri))
-            )
+            loc = get_uri_location(self.uri)
+            paths = filesystem.get_file_info(fs.FileSelector(loc))
         except FileNotFoundError:
             # It is ok if the file does not exist since it will be created
             paths = []
@@ -373,7 +372,7 @@ class LanceDBConnection(DBConnection):
         """
         try:
             filesystem, path = fs_from_uri(self.uri)
-            table_path = os.path.join(path, name + ".lance")
+            table_path = join_uri(path, name + ".lance")
             filesystem.delete_dir(table_path)
         except FileNotFoundError:
             if not ignore_missing:

python/lancedb/fts.py

@@ -75,8 +75,14 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
         The number of rows indexed
     """
     # first check the fields exist and are string or large string type
+    nested = []
     for name in fields:
-        f = table.schema.field(name)  # raises KeyError if not found
+        try:
+            f = table.schema.field(name)  # raises KeyError if not found
+        except KeyError:
+            f = resolve_path(table.schema, name)
+            nested.append(name)
+
         if not pa.types.is_string(f.type) and not pa.types.is_large_string(f.type):
             raise TypeError(f"Field {name} is not a string type")
 
@@ -85,7 +91,16 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
     # write data into index
     dataset = table.to_lance()
     row_id = 0
+
+    max_nested_level = 0
+    if len(nested) > 0:
+        max_nested_level = max([len(name.split(".")) for name in nested])
+
     for b in dataset.to_batches(columns=fields):
+        if max_nested_level > 0:
+            b = pa.Table.from_batches([b])
+            for _ in range(max_nested_level - 1):
+                b = b.flatten()
         for i in range(b.num_rows):
             doc = tantivy.Document()
             doc.add_integer("doc_id", row_id)
@@ -98,6 +113,30 @@ def populate_index(index: tantivy.Index, table: LanceTable, fields: List[str]) -
     return row_id
 
 
+def resolve_path(schema, field_name: str) -> pa.Field:
+    """
+    Resolve a nested field path to a list of field names
+
+    Parameters
+    ----------
+    field_name : str
+        The field name to resolve
+
+    Returns
+    -------
+    List[str]
+        The resolved path
+    """
+    path = field_name.split(".")
+    field = schema.field(path.pop(0))
+    for segment in path:
+        if pa.types.is_struct(field.type):
+            field = field.type.field(segment)
+        else:
+            raise KeyError(f"field {field_name} not found in schema {schema}")
+    return field
+
+
 def search_index(
     index: tantivy.Index, query: str, limit: int = 10
 ) -> Tuple[Tuple[int], Tuple[float]]:

python/lancedb/pydantic.py

@@ -348,3 +348,20 @@ def get_extras(field_info: pydantic.fields.FieldInfo, key: str) -> Any:
     if PYDANTIC_VERSION.major >= 2:
         return (field_info.json_schema_extra or {}).get(key)
     return (field_info.field_info.extra or {}).get("json_schema_extra", {}).get(key)
+
+
+if PYDANTIC_VERSION.major < 2:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.dict()
+
+else:
+
+    def model_to_dict(model: pydantic.BaseModel) -> Dict[str, Any]:
+        """
+        Convert a Pydantic model to a dictionary.
+        """
+        return model.model_dump()

python/lancedb/query.py

@@ -185,14 +185,40 @@ class LanceQueryBuilder(ABC):
         """
         return self.to_pandas()
 
-    def to_pandas(self) -> "pd.DataFrame":
+    def to_pandas(self, flatten: Optional[Union[int, bool]] = None) -> "pd.DataFrame":
         """
         Execute the query and return the results as a pandas DataFrame.
         In addition to the selected columns, LanceDB also returns a vector
         and also the "_distance" column which is the distance between the query
         vector and the returned vector.
+
+        Parameters
+        ----------
+        flatten: Optional[Union[int, bool]]
+            If flatten is True, flatten all nested columns.
+            If flatten is an integer, flatten the nested columns up to the
+            specified depth.
+            If unspecified, do not flatten the nested columns.
         """
-        return self.to_arrow().to_pandas()
+        tbl = self.to_arrow()
+        if flatten is True:
+            while True:
+                tbl = tbl.flatten()
+                has_struct = False
+                # loop through all columns to check if there is any struct column
+                if any(pa.types.is_struct(col.type) for col in tbl.schema):
+                    continue
+                else:
+                    break
+        elif isinstance(flatten, int):
+            if flatten <= 0:
+                raise ValueError(
+                    "Please specify a positive integer for flatten or the boolean value `True`"
+                )
+            while flatten > 0:
+                tbl = tbl.flatten()
+                flatten -= 1
+        return tbl.to_pandas()
 
     @abstractmethod
     def to_arrow(self) -> pa.Table:

python/lancedb/remote/__init__.py

@@ -18,6 +18,8 @@ import attrs
 import pyarrow as pa
 from pydantic import BaseModel
 
+from lancedb.common import VECTOR_COLUMN_NAME
+
 __all__ = ["LanceDBClient", "VectorQuery", "VectorQueryResult"]
 
 
@@ -43,6 +45,8 @@ class VectorQuery(BaseModel):
 
     refine_factor: Optional[int] = None
 
+    vector_column: str = VECTOR_COLUMN_NAME
+
 
 @attrs.define
 class VectorQueryResult:

python/lancedb/remote/table.py

@@ -13,7 +13,7 @@
 
 import uuid
 from functools import cached_property
-from typing import Optional, Union
+from typing import Dict, Optional, Union
 
 import pyarrow as pa
 from lance import json_to_schema
@@ -22,6 +22,7 @@ from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
 
 from ..query import LanceVectorQueryBuilder
 from ..table import Query, Table, _sanitize_data
+from ..util import value_to_sql
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE
 from .db import RemoteDBConnection
@@ -273,3 +274,65 @@ class RemoteTable(Table):
         self._conn._loop.run_until_complete(
             self._conn._client.post(f"/v1/table/{self._name}/delete/", data=payload)
         )
+
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
+        """
+        This can be used to update zero to all rows depending on how many
+        rows match the where clause.
+
+        Parameters
+        ----------
+        where: str, optional
+            The SQL where clause to use when updating rows. For example, 'x = 2'
+            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+        values: dict, optional
+            The values to update. The keys are the column names and the values
+            are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> data = [
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
+        ... ]
+        >>> db = lancedb.connect("db://...", api_key="...", region="...") # doctest: +SKIP
+        >>> table = db.create_table("my_table", data) # doctest: +SKIP
+        >>> table.to_pandas() # doctest: +SKIP
+           x      vector # doctest: +SKIP
+        0  1  [1.0, 2.0] # doctest: +SKIP
+        1  2  [3.0, 4.0] # doctest: +SKIP
+        2  3  [5.0, 6.0] # doctest: +SKIP
+        >>> table.update(where="x = 2", values={"vector": [10, 10]}) # doctest: +SKIP
+        >>> table.to_pandas() # doctest: +SKIP
+           x        vector # doctest: +SKIP
+        0  1    [1.0, 2.0] # doctest: +SKIP
+        1  3    [5.0, 6.0] # doctest: +SKIP
+        2  2  [10.0, 10.0] # doctest: +SKIP
+
+        """
+        if values is not None and values_sql is not None:
+            raise ValueError("Only one of values or values_sql can be provided")
+        if values is None and values_sql is None:
+            raise ValueError("Either values or values_sql must be provided")
+
+        if values is not None:
+            updates = [[k, value_to_sql(v)] for k, v in values.items()]
+        else:
+            updates = [[k, v] for k, v in values_sql.items()]
+
+        payload = {"predicate": where, "updates": updates}
+        self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/update/", data=payload)
+        )

python/lancedb/table.py

@@ -17,20 +17,21 @@ import inspect
 import os
 from abc import ABC, abstractmethod
 from functools import cached_property
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 
 import lance
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
+import pyarrow.fs as pa_fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
 from .embeddings import EmbeddingFunctionConfig, EmbeddingFunctionRegistry
-from .pydantic import LanceModel
+from .pydantic import LanceModel, model_to_dict
 from .query import LanceQueryBuilder, Query
-from .util import fs_from_uri, safe_import_pandas
+from .util import fs_from_uri, safe_import_pandas, value_to_sql, join_uri
 from .utils.events import register_event
 
 if TYPE_CHECKING:
@@ -53,8 +54,10 @@ def _sanitize_data(
         # convert to list of dict if data is a bunch of LanceModels
         if isinstance(data[0], LanceModel):
             schema = data[0].__class__.to_arrow_schema()
-            data = [dict(d) for d in data]
-        data = pa.Table.from_pylist(data)
+            data = [model_to_dict(d) for d in data]
+            data = pa.Table.from_pylist(data, schema=schema)
+        else:
+            data = pa.Table.from_pylist(data)
     elif isinstance(data, dict):
         data = vec_to_table(data)
     elif pd is not None and isinstance(data, pd.DataFrame):
@@ -394,14 +397,6 @@ class LanceTable(Table):
         self.name = name
         self._version = version
 
-    def _reset_dataset(self, version=None):
-        try:
-            if "_dataset" in self.__dict__:
-                del self.__dict__["_dataset"]
-            self._version = version
-        except AttributeError:
-            pass
-
     @property
     def schema(self) -> pa.Schema:
         """Return the schema of the table.
@@ -410,16 +405,16 @@ class LanceTable(Table):
         -------
         pa.Schema
             A PyArrow schema object."""
-        return self._dataset.schema
+        return self.to_lance().schema
 
     def list_versions(self):
         """List all versions of the table"""
-        return self._dataset.versions()
+        return self.to_lance().versions()
 
     @property
     def version(self) -> int:
         """Get the current version of the table"""
-        return self._dataset.version
+        return self.to_lance().version
 
     def checkout(self, version: int):
         """Checkout a version of the table. This is an in-place operation.
@@ -452,14 +447,12 @@ class LanceTable(Table):
                vector    type
         0  [1.1, 0.9]  vector
         """
-        max_ver = max([v["version"] for v in self._dataset.versions()])
+        max_ver = max([v["version"] for v in self.to_lance().versions()])
         if version < 1 or version > max_ver:
             raise ValueError(f"Invalid version {version}")
-        self._reset_dataset(version=version)
 
         try:
-            # Accessing the property updates the cached value
-            _ = self._dataset
+            self.to_lance().checkout(version)
         except Exception as e:
             if "not found" in str(e):
                 raise ValueError(
@@ -502,7 +495,7 @@ class LanceTable(Table):
         >>> len(table.list_versions())
         4
         """
-        max_ver = max([v["version"] for v in self._dataset.versions()])
+        max_ver = max([v["version"] for v in self.to_lance().versions()])
         if version is None:
             version = self.version
         elif version < 1 or version > max_ver:
@@ -514,11 +507,10 @@ class LanceTable(Table):
             # no-op if restoring the latest version
             return
 
-        self._dataset.restore()
-        self._reset_dataset()
+        self.to_lance().restore()
 
     def __len__(self):
-        return self._dataset.count_rows()
+        return self.to_lance().count_rows()
 
     def __repr__(self) -> str:
         return f"LanceTable({self.name})"
@@ -528,7 +520,7 @@ class LanceTable(Table):
 
     def head(self, n=5) -> pa.Table:
         """Return the first n rows of the table."""
-        return self._dataset.head(n)
+        return self.to_lance().head(n)
 
     def to_pandas(self) -> "pd.DataFrame":
         """Return the table as a pandas DataFrame.
@@ -545,11 +537,11 @@ class LanceTable(Table):
         Returns
         -------
         pa.Table"""
-        return self._dataset.to_table()
+        return self.to_lance().to_table()
 
     @property
     def _dataset_uri(self) -> str:
-        return os.path.join(self._conn.uri, f"{self.name}.lance")
+        return join_uri(self._conn.uri, f"{self.name}.lance")
 
     def create_index(
         self,
@@ -572,10 +564,11 @@ class LanceTable(Table):
             accelerator=accelerator,
             index_cache_size=index_cache_size,
         )
-        self._reset_dataset()
         register_event("create_index")
 
-    def create_fts_index(self, field_names: Union[str, List[str]]):
+    def create_fts_index(
+        self, field_names: Union[str, List[str]], *, replace: bool = False
+    ):
         """Create a full-text search index on the table.
 
         Warning - this API is highly experimental and is highly likely to change
@@ -585,17 +578,35 @@ class LanceTable(Table):
         ----------
         field_names: str or list of str
             The name(s) of the field to index.
+        replace: bool, default False
+            If True, replace the existing index if it exists. Note that this is
+            not yet an atomic operation; the index will be temporarily
+            unavailable while the new index is being created.
         """
         from .fts import create_index, populate_index
 
         if isinstance(field_names, str):
             field_names = [field_names]
+
+        fs, path = fs_from_uri(self._get_fts_index_path())
+        index_exists = fs.get_file_info(path).type != pa_fs.FileType.NotFound
+        if index_exists:
+            if not replace:
+                raise ValueError(
+                    f"Index already exists. Use replace=True to overwrite."
+                )
+            try:
+                fs.delete_dir(path)
+            except FileNotFoundError as e:
+                if "Cannot get information for path" in str(e):
+                    pass
+
         index = create_index(self._get_fts_index_path(), field_names)
         populate_index(index, self, field_names)
         register_event("create_fts_index")
 
     def _get_fts_index_path(self):
-        return os.path.join(self._dataset_uri, "_indices", "tantivy")
+        return join_uri(self._dataset_uri, "_indices", "tantivy")
 
     @cached_property
     def _dataset(self) -> LanceDataset:
@@ -643,8 +654,7 @@ class LanceTable(Table):
             on_bad_vectors=on_bad_vectors,
             fill_value=fill_value,
         )
-        lance.write_dataset(data, self._dataset_uri, schema=self.schema, mode=mode)
-        self._reset_dataset()
+        self.to_lance().write(data, mode=mode)
         register_event("add")
 
     def merge(
@@ -705,10 +715,9 @@ class LanceTable(Table):
             other_table = other_table.to_lance()
         if isinstance(other_table, LanceDataset):
             other_table = other_table.to_table()
-        self._dataset.merge(
+        self.to_lance().merge(
             other_table, left_on=left_on, right_on=right_on, schema=schema
         )
-        self._reset_dataset()
         register_event("merge")
 
     @cached_property
@@ -911,32 +920,37 @@ class LanceTable(Table):
         return tbl
 
     def delete(self, where: str):
-        self._dataset.delete(where)
+        self.to_lance().delete(where)
 
-    def update(self, where: str, values: dict):
+    def update(
+        self,
+        where: Optional[str] = None,
+        values: Optional[dict] = None,
+        *,
+        values_sql: Optional[Dict[str, str]] = None,
+    ):
         """
-        EXPERIMENTAL: Update rows in the table (not threadsafe).
-
         This can be used to update zero to all rows depending on how many
         rows match the where clause.
 
         Parameters
         ----------
-        where: str
+        where: str, optional
             The SQL where clause to use when updating rows. For example, 'x = 2'
             or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
-        values: dict
+        values: dict, optional
             The values to update. The keys are the column names and the values
             are the values to set.
+        values_sql: dict, optional
+            The values to update, expressed as SQL expression strings. These can
+            reference existing columns. For example, {"x": "x + 1"} will increment
+            the x column by 1.
 
         Examples
         --------
         >>> import lancedb
-        >>> data = [
-        ...    {"x": 1, "vector": [1, 2]},
-        ...    {"x": 2, "vector": [3, 4]},
-        ...    {"x": 3, "vector": [5, 6]}
-        ... ]
+        >>> import pandas as pd
+        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1, 2], [3, 4], [5, 6]]})
         >>> db = lancedb.connect("./.lancedb")
         >>> table = db.create_table("my_table", data)
         >>> table.to_pandas()
@@ -952,19 +966,15 @@ class LanceTable(Table):
         2  2  [10.0, 10.0]
 
         """
-        orig_data = self._dataset.to_table(filter=where).combine_chunks()
-        if len(orig_data) == 0:
-            return
-        for col, val in values.items():
-            i = orig_data.column_names.index(col)
-            if i < 0:
-                raise ValueError(f"Column {col} does not exist")
-            orig_data = orig_data.set_column(
-                i, col, pa.array([val] * len(orig_data), type=orig_data[col].type)
-            )
-        self.delete(where)
-        self.add(orig_data, mode="append")
-        self._reset_dataset()
+        if values is not None and values_sql is not None:
+            raise ValueError("Only one of values or values_sql can be provided")
+        if values is None and values_sql is None:
+            raise ValueError("Either values or values_sql must be provided")
+
+        if values is not None:
+            values_sql = {k: value_to_sql(v) for k, v in values.items()}
+
+        self.to_lance().update(values_sql, where)
         register_event("update")
 
     def _execute_query(self, query: Query) -> pa.Table:

python/lancedb/util.py

@@ -12,9 +12,13 @@
 #  limitations under the License.
 
 import os
-from typing import Tuple
+from datetime import date, datetime
+from functools import singledispatch
+import pathlib
+from typing import Tuple, Union
 from urllib.parse import urlparse
 
+import numpy as np
 import pyarrow.fs as pa_fs
 
 
@@ -59,6 +63,12 @@ def get_uri_location(uri: str) -> str:
     str: Location part of the URL, without scheme
     """
     parsed = urlparse(uri)
+    if len(parsed.scheme) == 1:
+        # Windows drive names are parsed as the scheme
+        # e.g. "c:\path" -> ParseResult(scheme="c", netloc="", path="/path", ...)
+        # So we add special handling here for schemes that are a single character
+        return uri
+
     if not parsed.netloc:
         return parsed.path
     else:
@@ -81,6 +91,29 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
     return pa_fs.FileSystem.from_uri(uri)
 
 
+def join_uri(base: Union[str, pathlib.Path], *parts: str) -> str:
+    """
+    Join a URI with multiple parts, handles both local and remote paths
+
+    Parameters
+    ----------
+    base : str
+        The base URI
+    parts : str
+        The parts to join to the base URI, each separated by the
+        appropriate path separator for the URI scheme and OS
+    """
+    if isinstance(base, pathlib.Path):
+        return base.joinpath(*parts)
+    base = str(base)
+    if get_uri_scheme(base) == "file":
+        # using pathlib for local paths make this windows compatible
+        # `get_uri_scheme` returns `file` for windows drive names (e.g. `c:\path`)
+        return str(pathlib.Path(base, *parts))
+    # for remote paths, just use os.path.join
+    return "/".join([p.rstrip("/") for p in [base, *parts]])
+
+
 def safe_import_pandas():
     try:
         import pandas as pd
@@ -88,3 +121,53 @@ def safe_import_pandas():
         return pd
     except ImportError:
         return None
+
+
+@singledispatch
+def value_to_sql(value):
+    raise NotImplementedError("SQL conversion is not implemented for this type")
+
+
+@value_to_sql.register(str)
+def _(value: str):
+    return f"'{value}'"
+
+
+@value_to_sql.register(int)
+def _(value: int):
+    return str(value)
+
+
+@value_to_sql.register(float)
+def _(value: float):
+    return str(value)
+
+
+@value_to_sql.register(bool)
+def _(value: bool):
+    return str(value).upper()
+
+
+@value_to_sql.register(type(None))
+def _(value: type(None)):
+    return "NULL"
+
+
+@value_to_sql.register(datetime)
+def _(value: datetime):
+    return f"'{value.isoformat()}'"
+
+
+@value_to_sql.register(date)
+def _(value: date):
+    return f"'{value.isoformat()}'"
+
+
+@value_to_sql.register(list)
+def _(value: list):
+    return "[" + ", ".join(map(value_to_sql, value)) + "]"
+
+
+@value_to_sql.register(np.ndarray)
+def _(value: np.ndarray):
+    return value_to_sql(value.tolist())

python/pyproject.toml

@@ -1,12 +1,12 @@
 [project]
 name = "lancedb"
-version = "0.3.4"
+version = "0.4.0"
 dependencies = [
     "deprecation",
-    "pylance==0.8.17",
+    "pylance==0.9.1",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
-    "tqdm>=4.1.0",
+    "tqdm>=4.27.0",
     "aiohttp",
     "pydantic>=1.10",
     "attrs>=21.3.0",

python/tests/test_fts.py

@@ -43,7 +43,15 @@ def table(tmp_path) -> ldb.table.LanceTable:
         for _ in range(100)
     ]
     table = db.create_table(
-        "test", data=pd.DataFrame({"vector": vectors, "text": text, "text2": text})
+        "test",
+        data=pd.DataFrame(
+            {
+                "vector": vectors,
+                "text": text,
+                "text2": text,
+                "nested": [{"text": t} for t in text],
+            }
+        ),
     )
     return table
 
@@ -75,6 +83,24 @@ def test_create_index_from_table(tmp_path, table):
     assert len(df) == 10
     assert "text" in df.columns
 
+    # Check whether it can be updated
+    table.add(
+        [
+            {
+                "vector": np.random.randn(128),
+                "text": "gorilla",
+                "text2": "gorilla",
+                "nested": {"text": "gorilla"},
+            }
+        ]
+    )
+
+    table.create_fts_index("text", replace=True)
+    assert len(table.search("gorilla").limit(1).to_pandas()) == 1
+
+    with pytest.raises(ValueError, match="already exists"):
+        table.create_fts_index("text")
+
 
 def test_create_index_multiple_columns(tmp_path, table):
     table.create_fts_index(["text", "text2"])
@@ -89,3 +115,9 @@ def test_empty_rs(tmp_path, table, mocker):
     mocker.patch("lancedb.fts.search_index", return_value=([], []))
     df = table.search("puppy").limit(10).to_pandas()
     assert len(df) == 0
+
+
+def test_nested_schema(tmp_path, table):
+    table.create_fts_index("nested.text")
+    rs = table.search("puppy").limit(10).to_list()
+    assert len(rs) == 10

python/tests/test_table.py

@@ -12,7 +12,7 @@
 #  limitations under the License.
 
 import functools
-from datetime import timedelta
+from datetime import date, datetime, timedelta
 from pathlib import Path
 from typing import List
 from unittest.mock import PropertyMock, patch
@@ -22,6 +22,7 @@ import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pytest
+from pydantic import BaseModel
 
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.db import LanceDBConnection
@@ -141,14 +142,44 @@ def test_add(db):
 
 
 def test_add_pydantic_model(db):
-    class TestModel(LanceModel):
-        vector: Vector(16)
-        li: List[int]
+    # https://github.com/lancedb/lancedb/issues/562
 
-    data = TestModel(vector=list(range(16)), li=[1, 2, 3])
-    table = LanceTable.create(db, "test", data=[data])
-    assert len(table) == 1
-    assert table.schema == TestModel.to_arrow_schema()
+    class Metadata(BaseModel):
+        source: str
+        timestamp: datetime
+
+    class Document(BaseModel):
+        content: str
+        meta: Metadata
+
+    class LanceSchema(LanceModel):
+        id: str
+        vector: Vector(2)
+        li: List[int]
+        payload: Document
+
+    tbl = LanceTable.create(db, "mytable", schema=LanceSchema, mode="overwrite")
+    assert tbl.schema == LanceSchema.to_arrow_schema()
+
+    # add works
+    expected = LanceSchema(
+        id="id",
+        vector=[0.0, 0.0],
+        li=[1, 2, 3],
+        payload=Document(
+            content="foo", meta=Metadata(source="bar", timestamp=datetime.now())
+        ),
+    )
+    tbl.add([expected])
+
+    result = tbl.search([0.0, 0.0]).limit(1).to_pydantic(LanceSchema)[0]
+    assert result == expected
+
+    flattened = tbl.search([0.0, 0.0]).limit(1).to_pandas(flatten=1)
+    assert len(flattened.columns) == 6  # _distance is automatically added
+
+    really_flattened = tbl.search([0.0, 0.0]).limit(1).to_pandas(flatten=True)
+    assert len(really_flattened.columns) == 7
 
 
 def _add(table, schema):
@@ -195,39 +226,38 @@ def test_versioning(db):
 
 
 def test_create_index_method():
-    with patch.object(LanceTable, "_reset_dataset", return_value=None):
-        with patch.object(
-            LanceTable, "_dataset", new_callable=PropertyMock
-        ) as mock_dataset:
-            # Setup mock responses
-            mock_dataset.return_value.create_index.return_value = None
+    with patch.object(
+        LanceTable, "_dataset", new_callable=PropertyMock
+    ) as mock_dataset:
+        # Setup mock responses
+        mock_dataset.return_value.create_index.return_value = None
 
-            # Create a LanceTable object
-            connection = LanceDBConnection(uri="mock.uri")
-            table = LanceTable(connection, "test_table")
+        # Create a LanceTable object
+        connection = LanceDBConnection(uri="mock.uri")
+        table = LanceTable(connection, "test_table")
 
-            # Call the create_index method
-            table.create_index(
-                metric="L2",
-                num_partitions=256,
-                num_sub_vectors=96,
-                vector_column_name="vector",
-                replace=True,
-                index_cache_size=256,
-            )
+        # Call the create_index method
+        table.create_index(
+            metric="L2",
+            num_partitions=256,
+            num_sub_vectors=96,
+            vector_column_name="vector",
+            replace=True,
+            index_cache_size=256,
+        )
 
-            # Check that the _dataset.create_index method was called
-            # with the right parameters
-            mock_dataset.return_value.create_index.assert_called_once_with(
-                column="vector",
-                index_type="IVF_PQ",
-                metric="L2",
-                num_partitions=256,
-                num_sub_vectors=96,
-                replace=True,
-                accelerator=None,
-                index_cache_size=256,
-            )
+        # Check that the _dataset.create_index method was called
+        # with the right parameters
+        mock_dataset.return_value.create_index.assert_called_once_with(
+            column="vector",
+            index_type="IVF_PQ",
+            metric="L2",
+            num_partitions=256,
+            num_sub_vectors=96,
+            replace=True,
+            accelerator=None,
+            index_cache_size=256,
+        )
 
 
 def test_add_with_nans(db):
@@ -348,14 +378,79 @@ def test_update(db):
     assert len(table) == 2
     assert len(table.list_versions()) == 2
     table.update(where="id=0", values={"vector": [1.1, 1.1]})
-    assert len(table.list_versions()) == 4
-    assert table.version == 4
+    assert len(table.list_versions()) == 3
+    assert table.version == 3
     assert len(table) == 2
     v = table.to_arrow()["vector"].combine_chunks()
     v = v.values.to_numpy().reshape(2, 2)
     assert np.allclose(v, np.array([[1.2, 1.9], [1.1, 1.1]]))
 
 
+def test_update_types(db):
+    table = LanceTable.create(
+        db,
+        "my_table",
+        data=[
+            {
+                "id": 0,
+                "str": "foo",
+                "float": 1.1,
+                "timestamp": datetime(2021, 1, 1),
+                "date": date(2021, 1, 1),
+                "vector1": [1.0, 0.0],
+                "vector2": [1.0, 1.0],
+            }
+        ],
+    )
+    # Update with SQL
+    table.update(
+        values_sql=dict(
+            id="1",
+            str="'bar'",
+            float="2.2",
+            timestamp="TIMESTAMP '2021-01-02 00:00:00'",
+            date="DATE '2021-01-02'",
+            vector1="[2.0, 2.0]",
+            vector2="[3.0, 3.0]",
+        )
+    )
+    actual = table.to_arrow().to_pylist()[0]
+    expected = dict(
+        id=1,
+        str="bar",
+        float=2.2,
+        timestamp=datetime(2021, 1, 2),
+        date=date(2021, 1, 2),
+        vector1=[2.0, 2.0],
+        vector2=[3.0, 3.0],
+    )
+    assert actual == expected
+
+    # Update with values
+    table.update(
+        values=dict(
+            id=2,
+            str="baz",
+            float=3.3,
+            timestamp=datetime(2021, 1, 3),
+            date=date(2021, 1, 3),
+            vector1=[3.0, 3.0],
+            vector2=np.array([4.0, 4.0]),
+        )
+    )
+    actual = table.to_arrow().to_pylist()[0]
+    expected = dict(
+        id=2,
+        str="baz",
+        float=3.3,
+        timestamp=datetime(2021, 1, 3),
+        date=date(2021, 1, 3),
+        vector1=[3.0, 3.0],
+        vector2=[4.0, 4.0],
+    )
+    assert actual == expected
+
+
 def test_create_with_embedding_function(db):
     class MyTable(LanceModel):
         text: str

python/tests/test_util.py

@@ -11,7 +11,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from lancedb.util import get_uri_scheme
+import os
+import pathlib
+
+import pytest
+
+from lancedb.util import get_uri_scheme, join_uri
 
 
 def test_normalize_uri():
@@ -28,3 +33,55 @@ def test_normalize_uri():
     for uri, expected_scheme in zip(uris, schemes):
         parsed_scheme = get_uri_scheme(uri)
         assert parsed_scheme == expected_scheme
+
+
+def test_join_uri_remote():
+    schemes = ["s3", "az", "gs"]
+    for scheme in schemes:
+        expected = f"{scheme}://bucket/path/to/table.lance"
+        base_uri = f"{scheme}://bucket/path/to/"
+        parts = ["table.lance"]
+        assert join_uri(base_uri, *parts) == expected
+
+        base_uri = f"{scheme}://bucket"
+        parts = ["path", "to", "table.lance"]
+        assert join_uri(base_uri, *parts) == expected
+
+
+# skip this test if on windows
+@pytest.mark.skipif(os.name == "nt", reason="Windows paths are not POSIX")
+def test_join_uri_posix():
+    for base in [
+        # relative path
+        "relative/path",
+        "relative/path/",
+        # an absolute path
+        "/absolute/path",
+        "/absolute/path/",
+        # a file URI
+        "file:///absolute/path",
+        "file:///absolute/path/",
+    ]:
+        joined = join_uri(base, "table.lance")
+        assert joined == str(pathlib.Path(base) / "table.lance")
+        joined = join_uri(pathlib.Path(base), "table.lance")
+        assert joined == pathlib.Path(base) / "table.lance"
+
+
+# skip this test if not on windows
+@pytest.mark.skipif(os.name != "nt", reason="Windows paths are not POSIX")
+def test_local_join_uri_windows():
+    # https://learn.microsoft.com/en-us/dotnet/standard/io/file-path-formats
+    for base in [
+        # windows relative path
+        "relative\\path",
+        "relative\\path\\",
+        # windows absolute path from current drive
+        "c:\\absolute\\path",
+        # relative path from root of current drive
+        "\\relative\\path",
+    ]:
+        joined = join_uri(base, "table.lance")
+        assert joined == str(pathlib.Path(base) / "table.lance")
+        joined = join_uri(pathlib.Path(base), "table.lance")
+        assert joined == pathlib.Path(base) / "table.lance"

rust/ffi/node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.3.9"
+version = "0.4.0"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"

rust/ffi/node/src/error.rs

@@ -23,7 +23,7 @@ pub enum Error {
     #[snafu(display("column '{name}' is missing"))]
     MissingColumn { name: String },
     #[snafu(display("{name}: {message}"))]
-    RangeError { name: String, message: String },
+    OutOfRange { name: String, message: String },
     #[snafu(display("{index_type} is not a valid index type"))]
     InvalidIndexType { index_type: String },
 

rust/ffi/node/src/index/vector.rs

@@ -65,12 +65,10 @@ fn get_index_params_builder(
             obj.get_opt::<JsString, _, _>(cx, "index_name")?
                 .map(|s| index_builder.index_name(s.value(cx)));
 
-            obj.get_opt::<JsString, _, _>(cx, "metric_type")?
-                .map(|s| MetricType::try_from(s.value(cx).as_str()))
-                .map(|mt| {
-                    let metric_type = mt.unwrap();
-                    index_builder.metric_type(metric_type);
-                });
+            if let Some(metric_type) = obj.get_opt::<JsString, _, _>(cx, "metric_type")? {
+                let metric_type = MetricType::try_from(metric_type.value(cx).as_str()).unwrap();
+                index_builder.metric_type(metric_type);
+            }
 
             let num_partitions = obj.get_opt_usize(cx, "num_partitions")?;
             let max_iters = obj.get_opt_usize(cx, "max_iters")?;
@@ -85,23 +83,29 @@ fn get_index_params_builder(
                 index_builder.ivf_params(ivf_params)
             });
 
-            obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")?
-                .map(|s| pq_params.use_opq = s.value(cx));
+            if let Some(use_opq) = obj.get_opt::<JsBoolean, _, _>(cx, "use_opq")? {
+                pq_params.use_opq = use_opq.value(cx);
+            }
 
-            obj.get_opt_usize(cx, "num_sub_vectors")?
-                .map(|s| pq_params.num_sub_vectors = s);
+            if let Some(num_sub_vectors) = obj.get_opt_usize(cx, "num_sub_vectors")? {
+                pq_params.num_sub_vectors = num_sub_vectors;
+            }
 
-            obj.get_opt_usize(cx, "num_bits")?
-                .map(|s| pq_params.num_bits = s);
+            if let Some(num_bits) = obj.get_opt_usize(cx, "num_bits")? {
+                pq_params.num_bits = num_bits;
+            }
 
-            obj.get_opt_usize(cx, "max_iters")?
-                .map(|s| pq_params.max_iters = s);
+            if let Some(max_iters) = obj.get_opt_usize(cx, "max_iters")? {
+                pq_params.max_iters = max_iters;
+            }
 
-            obj.get_opt_usize(cx, "max_opq_iters")?
-                .map(|s| pq_params.max_opq_iters = s);
+            if let Some(max_opq_iters) = obj.get_opt_usize(cx, "max_opq_iters")? {
+                pq_params.max_opq_iters = max_opq_iters;
+            }
 
-            obj.get_opt::<JsBoolean, _, _>(cx, "replace")?
-                .map(|s| index_builder.replace(s.value(cx)));
+            if let Some(replace) = obj.get_opt::<JsBoolean, _, _>(cx, "replace")? {
+                index_builder.replace(replace.value(cx));
+            }
 
             Ok(index_builder)
         }

rust/ffi/node/src/neon_ext/js_object_ext.rs

@@ -47,15 +47,15 @@ fn f64_to_u32_safe(n: f64, key: &str) -> Result<u32> {
     use conv::*;
 
     n.approx_as::<u32>().map_err(|e| match e {
-        FloatError::NegOverflow(_) => Error::RangeError {
+        FloatError::NegOverflow(_) => Error::OutOfRange {
             name: key.into(),
             message: "must be > 0".to_string(),
         },
-        FloatError::PosOverflow(_) => Error::RangeError {
+        FloatError::PosOverflow(_) => Error::OutOfRange {
             name: key.into(),
             message: format!("must be < {}", u32::MAX),
         },
-        FloatError::NotANumber(_) => Error::RangeError {
+        FloatError::NotANumber(_) => Error::OutOfRange {
             name: key.into(),
             message: "not a valid number".to_string(),
         },
@@ -66,15 +66,15 @@ fn f64_to_usize_safe(n: f64, key: &str) -> Result<usize> {
     use conv::*;
 
     n.approx_as::<usize>().map_err(|e| match e {
-        FloatError::NegOverflow(_) => Error::RangeError {
+        FloatError::NegOverflow(_) => Error::OutOfRange {
             name: key.into(),
             message: "must be > 0".to_string(),
         },
-        FloatError::PosOverflow(_) => Error::RangeError {
+        FloatError::PosOverflow(_) => Error::OutOfRange {
             name: key.into(),
             message: format!("must be < {}", usize::MAX),
         },
-        FloatError::NotANumber(_) => Error::RangeError {
+        FloatError::NotANumber(_) => Error::OutOfRange {
             name: key.into(),
             message: "not a valid number".to_string(),
         },

rust/ffi/node/src/query.rs

@@ -25,11 +25,11 @@ impl JsQuery {
         let limit = query_obj
             .get_opt::<JsNumber, _, _>(&mut cx, "_limit")?
             .map(|value| {
-                let limit = value.value(&mut cx) as u64;
-                if limit <= 0 {
+                let limit = value.value(&mut cx);
+                if limit <= 0.0 {
                     panic!("Limit must be a positive integer");
                 }
-                limit
+                limit as u64
             });
         let select = query_obj
             .get_opt::<JsArray, _, _>(&mut cx, "_select")?
@@ -73,7 +73,7 @@ impl JsQuery {
 
         rt.spawn(async move {
             let mut builder = table
-                .search(query.map(|q| Float32Array::from(q)))
+                .search(query.map(Float32Array::from))
                 .refine_factor(refine_factor)
                 .nprobes(nprobes)
                 .filter(filter)

rust/ffi/node/src/table.rs

@@ -45,7 +45,7 @@ impl JsTable {
         let table_name = cx.argument::<JsString>(0)?.value(&mut cx);
         let buffer = cx.argument::<JsBuffer>(1)?;
         let (batches, schema) =
-            arrow_buffer_to_record_batch(buffer.as_slice(&mut cx)).or_throw(&mut cx)?;
+            arrow_buffer_to_record_batch(buffer.as_slice(&cx)).or_throw(&mut cx)?;
 
         // Write mode
         let mode = match cx.argument::<JsString>(2)?.value(&mut cx).as_str() {
@@ -93,7 +93,7 @@ impl JsTable {
         let buffer = cx.argument::<JsBuffer>(0)?;
         let write_mode = cx.argument::<JsString>(1)?.value(&mut cx);
         let (batches, schema) =
-            arrow_buffer_to_record_batch(buffer.as_slice(&mut cx)).or_throw(&mut cx)?;
+            arrow_buffer_to_record_batch(buffer.as_slice(&cx)).or_throw(&mut cx)?;
         let rt = runtime(&mut cx)?;
         let channel = cx.channel();
         let mut table = js_table.table.clone();
@@ -186,7 +186,7 @@ impl JsTable {
                 .downcast_or_throw::<JsString, _>(&mut cx)?;
 
             let value = updates_arg
-                .get_value(&mut cx, property.clone())?
+                .get_value(&mut cx, property)?
                 .downcast_or_throw::<JsString, _>(&mut cx)?;
 
             let property = property.value(&mut cx);
@@ -216,7 +216,7 @@ impl JsTable {
                 .map(|(k, v)| (k.as_str(), v.as_str()))
                 .collect::<Vec<_>>();
 
-            let predicate = predicate.as_ref().map(|s| s.as_str());
+            let predicate = predicate.as_deref();
 
             let update_result = table.update(predicate, updates_arg).await;
             deferred.settle_with(&channel, move |mut cx| {

rust/vectordb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.3.9"
+version = "0.4.0"
 edition = "2021"
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"

rust/vectordb/src/io/object_store.rs

@@ -26,7 +26,7 @@ use futures::{stream::BoxStream, FutureExt, StreamExt};
 use lance::io::object_store::WrappingObjectStore;
 use object_store::{
     path::Path, Error, GetOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore,
-    Result,
+    PutOptions, PutResult, Result,
 };
 
 use async_trait::async_trait;
@@ -72,13 +72,28 @@ impl PrimaryOnly for Path {
 /// Note: this object store does not mirror writes to *.manifest files
 #[async_trait]
 impl ObjectStore for MirroringObjectStore {
-    async fn put(&self, location: &Path, bytes: Bytes) -> Result<()> {
+    async fn put(&self, location: &Path, bytes: Bytes) -> Result<PutResult> {
         if location.primary_only() {
             self.primary.put(location, bytes).await
         } else {
             self.secondary.put(location, bytes.clone()).await?;
-            self.primary.put(location, bytes).await?;
-            Ok(())
+            self.primary.put(location, bytes).await
+        }
+    }
+
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: Bytes,
+        options: PutOptions,
+    ) -> Result<PutResult> {
+        if location.primary_only() {
+            self.primary.put_opts(location, bytes, options).await
+        } else {
+            self.secondary
+                .put_opts(location, bytes.clone(), options.clone())
+                .await?;
+            self.primary.put_opts(location, bytes, options).await
         }
     }
 
@@ -129,8 +144,8 @@ impl ObjectStore for MirroringObjectStore {
         self.primary.delete(location).await
     }
 
-    async fn list(&self, prefix: Option<&Path>) -> Result<BoxStream<'_, Result<ObjectMeta>>> {
-        self.primary.list(prefix).await
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'_, Result<ObjectMeta>> {
+        self.primary.list(prefix)
     }
 
     async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
@@ -359,7 +374,9 @@ mod test {
         assert_eq!(t.count_rows().await.unwrap(), 100);
 
         let q = t
-            .search(Some(PrimitiveArray::from_iter_values(vec![0.1, 0.1, 0.1, 0.1])))
+            .search(Some(PrimitiveArray::from_iter_values(vec![
+                0.1, 0.1, 0.1, 0.1,
+            ])))
             .limit(10)
             .execute()
             .await

rust/vectordb/src/query.rs

@@ -25,6 +25,7 @@ use crate::error::Result;
 pub struct Query {
     pub dataset: Arc<Dataset>,
     pub query_vector: Option<Float32Array>,
+    pub column: String,
     pub limit: Option<usize>,
     pub filter: Option<String>,
     pub select: Option<Vec<String>>,
@@ -50,6 +51,7 @@ impl Query {
         Query {
             dataset,
             query_vector: vector,
+            column: crate::table::VECTOR_COLUMN_NAME.to_string(),
             limit: None,
             nprobes: 20,
             refine_factor: None,
@@ -71,7 +73,7 @@ impl Query {
 
         if let Some(query) = self.query_vector.as_ref() {
             // If there is a vector query, default to limit=10 if unspecified
-            scanner.nearest(crate::table::VECTOR_COLUMN_NAME, query, self.limit.unwrap_or(10))?;
+            scanner.nearest(&self.column, query, self.limit.unwrap_or(10))?;
         } else {
             // If there is no vector query, it's ok to not have a limit
             scanner.limit(self.limit.map(|limit| limit as i64), None)?;
@@ -87,6 +89,16 @@ impl Query {
         Ok(scanner.try_into_stream().await?)
     }
 
+    /// Set the column to query
+    ///
+    /// # Arguments
+    ///
+    /// * `column` - The column name
+    pub fn column(mut self, column: &str) -> Query {
+        self.column = column.into();
+        self
+    }
+
     /// Set the maximum number of results to return.
     ///
     /// # Arguments
@@ -176,7 +188,10 @@ mod tests {
     use std::sync::Arc;
 
     use super::*;
-    use arrow_array::{Float32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, cast::AsArray, Int32Array};
+    use arrow_array::{
+        cast::AsArray, Float32Array, Int32Array, RecordBatch, RecordBatchIterator,
+        RecordBatchReader,
+    };
     use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
     use futures::StreamExt;
     use lance::dataset::Dataset;
@@ -260,7 +275,7 @@ mod tests {
         let mut stream = result.expect("should have result");
         // should only have one batch
         while let Some(batch) = stream.next().await {
-            let b = batch.expect("should be Ok");            
+            let b = batch.expect("should be Ok");
             // cast arr into Int32Array
             let arr: &Int32Array = b["id"].as_primitive();
             assert!(arr.iter().all(|x| x.unwrap() % 2 == 0));