Bump version: 0.10.1-beta.0 → 0.10.1

Feel free to close if this is a distraction, but untyped keywords in
lance.connect is throwing pylance errors in strict mode.

<img width="683" alt="Screenshot 2024-07-11 at 1 21 04 PM"
src="https://github.com/lancedb/lancedb/assets/33043305/fe6cd4d9-4e59-413d-87f2-aabb9ff84cc4">

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.5.1"
+current_version = "0.7.0"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/build_linux_wheel/action.yml

@@ -46,6 +46,7 @@ runs:
       with:
         command: build
         working-directory: python
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         target: aarch64-unknown-linux-gnu
         manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}

.github/workflows/build_mac_wheel/action.yml

@@ -21,5 +21,6 @@ runs:
       with:
         command: build
         args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
         interpreter: 3.${{ inputs.python-minor-version }}

.github/workflows/build_windows_wheel/action.yml

@@ -26,6 +26,7 @@ runs:
       with:
         command: build
         args: ${{ inputs.args }}
+        docker-options: "-e PIP_EXTRA_INDEX_URL=https://pypi.fury.io/lancedb/"
         working-directory: python
     - uses: actions/upload-artifact@v3
       with:

.github/workflows/docs_test.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "warp-ubuntu-latest-x64-4x"
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -56,7 +56,7 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "warp-ubuntu-latest-x64-4x"
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/npm-publish.yml

@@ -3,7 +3,7 @@ name: NPM Publish
 on:
   push:
     tags:
-      - 'v*'
+      - "v*"
 
 jobs:
   node:
@@ -111,12 +111,11 @@ jobs:
             runner: ubuntu-latest
           - arch: aarch64
             # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
+            runner: warp-ubuntu-latest-arm64-4x
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
+      # To avoid OOM errors on ARM, we create a swap file.
-      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
       - name: Configure aarch64 build
         if: ${{ matrix.config.arch == 'aarch64' }}
         run: |
@@ -323,7 +322,7 @@ jobs:
       - name: Publish to NPM
         env:
           NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
-        # By default, things are published to the latest tag. This is what is 
+        # By default, things are published to the latest tag. This is what is
         # installed by default if the user does not specify a version. This is
         # good for stable releases, but for pre-releases, we want to publish to
         # the "preview" tag so they can install with `npm install lancedb@preview`.
@@ -368,7 +367,7 @@ jobs:
       - uses: ./.github/workflows/update_package_lock_nodejs
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-  
+
   gh-release:
     runs-on: ubuntu-latest
     permissions:

.github/workflows/python.yml

@@ -65,7 +65,7 @@ jobs:
           workspaces: python
       - name: Install
         run: |
-          pip install -e .[tests,dev,embeddings]
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
           pip install tantivy
           pip install mlx
       - name: Doctest
@@ -189,7 +189,7 @@ jobs:
       - name: Install lancedb
         run: |
           pip install "pydantic<2"
-          pip install -e .[tests]
+          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
           pip install tantivy
       - name: Run tests
         run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

.github/workflows/run_tests/action.yml

@@ -15,7 +15,7 @@ runs:
     - name: Install lancedb
       shell: bash
       run: |
-        pip3 install $(ls target/wheels/lancedb-*.whl)[tests,dev]
+        pip3 install --extra-index-url https://pypi.fury.io/lancedb/ $(ls target/wheels/lancedb-*.whl)[tests,dev]
     - name: Setup localstack for integration tests
       if: ${{ inputs.integration == 'true' }}
       shell: bash

.gitignore

@@ -4,6 +4,7 @@
 **/__pycache__
 .DS_Store
 venv
+.venv
 
 .vscode
 .zed

.pre-commit-config.yaml

@@ -14,8 +14,8 @@ repos:
   hooks:
     - id: local-biome-check
       name: biome check
-      entry: npx @biomejs/biome check --config-path nodejs/biome.json nodejs/
+      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
       language: system
       types: [text]
       files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*

Cargo.toml

@@ -20,11 +20,11 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.12.1", "features" = ["dynamodb"] }
+lance = { "version" = "=0.14.1", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.12.1" }
+lance-index = { "version" = "=0.14.1" }
-lance-linalg = { "version" = "=0.12.1" }
+lance-linalg = { "version" = "=0.14.1" }
-lance-testing = { "version" = "=0.12.1" }
+lance-testing = { "version" = "=0.14.1" }
-lance-datafusion = { "version" = "=0.12.1" }
+lance-datafusion = { "version" = "=0.14.1" }
 # Note that this one does not include pyarrow
 arrow = { version = "51.0", optional = false }
 arrow-array = "51.0"

docs/mkdocs.yml

@@ -57,6 +57,8 @@ plugins:
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
   - mkdocs-jupyter
+  - render_swagger:
+      allow_arbitrary_locations : true
 
 markdown_extensions:
   - admonition
@@ -100,12 +102,18 @@ nav:
               - Linear Combination Reranker: reranking/linear_combination.md
               - Cross Encoder Reranker: reranking/cross_encoder.md
               - ColBERT Reranker: reranking/colbert.md
+              - Jina Reranker: reranking/jina.md
               - OpenAI Reranker: reranking/openai.md
               - Building Custom Rerankers: reranking/custom_reranker.md
+              - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
           - Versioning & Reproducibility: notebooks/reproducibility.ipynb
           - Configuring Storage: guides/storage.md
-          - Sync -> Async Migration Guide: migration.md
+          - Migration Guide: migration.md
+          - Tuning retrieval performance:
+              - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+              - Reranking: guides/tuning_retrievers/2_reranking.md
+              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
       - 🧬 Managing embeddings:
           - Overview: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
@@ -120,8 +128,11 @@ nav:
           - DuckDB: python/duckdb.md
           - LangChain:
             - LangChain 🔗: integrations/langchain.md
+            - LangChain demo: notebooks/langchain_demo.ipynb
             - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-          - LlamaIndex 🦙: https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
+          - LlamaIndex 🦙:
+            - LlamaIndex docs: integrations/llamaIndex.md
+            - LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
           - Pydantic: python/pydantic.md
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
@@ -152,7 +163,8 @@ nav:
           - Overview: cloud/index.md
           - API reference:
               - 🐍 Python: python/saas-python.md
-              - 👾 JavaScript: javascript/saas-modules.md
+              - 👾 JavaScript: javascript/modules.md
+              - REST API: cloud/rest.md
 
   - Quick start: basic.md
   - Concepts:
@@ -175,12 +187,18 @@ nav:
           - Linear Combination Reranker: reranking/linear_combination.md
           - Cross Encoder Reranker: reranking/cross_encoder.md
           - ColBERT Reranker: reranking/colbert.md
+          - Jina Reranker: reranking/jina.md
           - OpenAI Reranker: reranking/openai.md
           - Building Custom Rerankers: reranking/custom_reranker.md
+          - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
       - Versioning & Reproducibility: notebooks/reproducibility.ipynb
       - Configuring Storage: guides/storage.md
-      - Sync -> Async Migration Guide: migration.md
+      - Migration Guide: migration.md
+      - Tuning retrieval performance:
+          - Choosing right query type: guides/tuning_retrievers/1_query_types.md
+          - Reranking: guides/tuning_retrievers/2_reranking.md
+          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
   - Managing Embeddings:
       - Overview: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
@@ -193,9 +211,9 @@ nav:
       - Pandas and PyArrow: python/pandas_and_pyarrow.md
       - Polars: python/polars_arrow.md
       - DuckDB: python/duckdb.md
-      - LangChain 🦜️🔗↗: https://python.langchain.com/docs/integrations/vectorstores/lancedb
+      - LangChain 🦜️🔗↗: integrations/langchain.md
       - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-      - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+      - LlamaIndex 🦙↗: integrations/llamaIndex.md
       - Pydantic: python/pydantic.md
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
@@ -219,7 +237,8 @@ nav:
       - Overview: cloud/index.md
       - API reference:
           - 🐍 Python: python/saas-python.md
-          - 👾 JavaScript: javascript/saas-modules.md
+          - 👾 JavaScript: javascript/modules.md
+          - REST API: cloud/rest.md
 
 extra_css:
   - styles/global.css

docs/openapi.yml

@@ -0,0 +1,487 @@
+openapi: 3.1.0
+info:
+  version: 1.0.0
+  title: LanceDB Cloud API
+  description: |
+    LanceDB Cloud API is a RESTful API that allows users to access and modify data stored in LanceDB Cloud.
+    Table actions are considered temporary resource creations and all use POST method.
+  contact:
+    name: LanceDB support
+    url: https://lancedb.com
+    email: contact@lancedb.com
+
+servers:
+  - url: https://{db}.{region}.api.lancedb.com
+    description: LanceDB Cloud REST endpoint.
+    variables:
+      db:
+        default: ""
+        description: the name of DB
+      region:
+        default: "us-east-1"
+        description: the service region of the DB
+
+security:
+  - key_auth: []
+
+components:
+  securitySchemes:
+    key_auth:
+      name: x-api-key
+      type: apiKey
+      in: header
+  parameters:
+    table_name:
+      name: name
+      in: path
+      description: name of the table
+      required: true
+      schema:
+        type: string
+  responses:
+    invalid_request:
+      description: Invalid request
+      content:
+        text/plain:
+          schema:
+            type: string
+    not_found:
+      description: Not found
+      content:
+        text/plain:
+          schema:
+            type: string
+    unauthorized:
+      description: Unauthorized
+      content:
+        text/plain:
+          schema:
+            type: string
+  requestBodies:
+    arrow_stream_buffer:
+      description: Arrow IPC stream buffer
+      required: true
+      content:
+        application/vnd.apache.arrow.stream:
+          schema:
+            type: string
+            format: binary
+
+paths:
+  /v1/table/:
+    get:
+      description: List tables, optionally, with pagination.
+      tags:
+        - Tables
+      summary: List Tables
+      operationId: listTables
+      parameters:
+        - name: limit
+          in: query
+          description: Limits the number of items to return.
+          schema:
+            type: integer
+        - name: page_token
+          in: query
+          description: Specifies the starting position of the next query
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Successfully returned a list of tables in the DB
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  tables:
+                    type: array
+                    items:
+                      type: string
+                  page_token:
+                    type: string
+
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/create/:
+    post:
+      description: Create a new table
+      summary: Create a new table
+      operationId: createTable
+      tags:
+        - Tables
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Table successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/query/:
+    post:
+      description: Vector Query
+      url: https://{db-uri}.{aws-region}.api.lancedb.com/v1/table/{name}/query/
+      tags:
+        - Data
+      summary: Vector Query
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                vector:
+                  type: FixedSizeList
+                  description: |
+                    The targetted vector to search for. Required.
+                vector_column:
+                  type: string
+                  description: |
+                    The column to query, it can be inferred from the schema if there is only one vector column.
+                prefilter:
+                  type: boolean
+                  description: |
+                    Whether to prefilter the data. Optional.
+                k:
+                  type: integer
+                  description: |
+                    The number of search results to return. Default is 10.
+                distance_type:
+                  type: string
+                  description: |
+                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
+                bypass_vector_index:
+                  type: boolean
+                  description: |
+                    Whether to bypass vector index. Optional.
+                filter:
+                  type: string
+                  description: |
+                    A filter expression that specifies the rows to query. Optional.
+                columns:
+                  type: array
+                  items:
+                    type: string
+                  description: |
+                    The columns to return. Optional.
+                nprobe:
+                  type: integer
+                  description: |
+                    The number of probes to use for search. Optional.
+                refine_factor:
+                  type: integer
+                  description: |
+                    The refine factor to use for search. Optional.
+                  default: null
+                fast_search:
+                  type: boolean
+                  description: |
+                    Whether to use fast search. Optional.
+                  default: false
+              required:
+                - vector
+
+      responses:
+        "200":
+          description: top k results if query is successfully executed
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  results:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        id:
+                          type: integer
+                        selected_col_1_to_return:
+                          type: col_1_type
+                        selected_col_n_to_return:
+                          type: col_n_type
+                        _distance:
+                          type: float
+
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/insert/:
+    post:
+      description: Insert new data to the Table.
+      tags:
+        - Data
+      operationId: insertData
+      summary: Insert new data.
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Insert successful
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/merge_insert/:
+    post:
+      description: Create a "merge insert" operation
+        This operation can add rows, update rows, and remove rows all in a single
+        transaction. See python method `lancedb.table.Table.merge_insert` for examples.
+      tags:
+        - Data
+      summary: Merge Insert
+      operationId: mergeInsert
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+        - name: on
+          in: query
+          description: |
+            The column to use as the primary key for the merge operation.
+          required: true
+          schema:
+            type: string
+        - name: when_matched_update_all
+          in: query
+          description: |
+            Rows that exist in both the source table (new data) and
+            the target table (old data) will be updated, replacing
+            the old row with the corresponding matching row.
+          required: false
+          schema:
+            type: boolean
+        - name: when_matched_update_all_filt
+          in: query
+          description: |
+            If present then only rows that satisfy the filter expression will
+            be updated
+          required: false
+          schema:
+            type: string
+        - name: when_not_matched_insert_all
+          in: query
+          description: |
+            Rows that exist only in the source table (new data) will be
+            inserted into the target table (old data).
+          required: false
+          schema:
+            type: boolean
+        - name: when_not_matched_by_source_delete
+          in: query
+          description: |
+            Rows that exist only in the target table (old data) will be
+            deleted. An optional condition (`when_not_matched_by_source_delete_filt`)
+            can be provided to limit what data is deleted.
+          required: false
+          schema:
+            type: boolean
+        - name: when_not_matched_by_source_delete_filt
+          in: query
+          description: |
+            The filter expression that specifies the rows to delete.
+          required: false
+          schema:
+            type: string
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Merge Insert successful
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/delete/:
+    post:
+      description: Delete rows from a table.
+      tags:
+        - Data
+      summary: Delete rows from a table
+      operationId: deleteData
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                predicate:
+                  type: string
+                  description: |
+                    A filter expression that specifies the rows to delete.
+      responses:
+        "200":
+          description: Delete successful
+        "401":
+          $ref: "#/components/responses/unauthorized"
+  /v1/table/{name}/drop/:
+    post:
+      description: Drop a table
+      tags:
+        - Tables
+      summary: Drop a table
+      operationId: dropTable
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Drop successful
+        "401":
+          $ref: "#/components/responses/unauthorized"
+
+  /v1/table/{name}/describe/:
+    post:
+      description: Describe a table and return Table Information.
+      tags:
+        - Tables
+      summary: Describe a table
+      operationId: describeTable
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      responses:
+        "200":
+          description: Table information
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  table:
+                    type: string
+                  version:
+                    type: integer
+                  schema:
+                    type: string
+                  stats:
+                    type: object
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/index/list/:
+    post:
+      description: List indexes of a table
+      tags:
+        - Tables
+      summary: List indexes of a table
+      operationId: listIndexes
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      responses:
+        "200":
+          description: Available list of indexes on the table.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  indexes:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        columns:
+                          type: array
+                          items:
+                            type: string
+                        index_name:
+                          type: string
+                        index_uuid:
+                          type: string
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/create_index/:
+    post:
+      description: Create vector index on a Table
+      tags:
+        - Tables
+      summary: Create vector index on a Table
+      operationId: createIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                column:
+                  type: string
+                metric_type:
+                  type: string
+                  nullable: false
+                  description: |
+                    The metric type to use for the index. L2, Cosine, Dot are supported.
+                index_type:
+                  type: string
+      responses:
+        "200":
+          description: Index successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/create_scalar_index/:
+    post:
+      description: Create a scalar index on a table
+      tags:
+        - Tables
+      summary: Create a scalar index on a table
+      operationId: createScalarIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                column:
+                  type: string
+                index_type:
+                  type: string
+                  required: false
+      responses:
+        "200":
+          description: Scalar Index successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"

docs/requirements.txt

@@ -2,4 +2,5 @@ mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
-pydantic
+mkdocs-render-swagger-plugin
+pydantic

docs/src/ann_indexes.md

@@ -38,13 +38,27 @@ Lance supports `IVF_PQ` index type by default.
     tbl.create_index(num_partitions=256, num_sub_vectors=96)
     ```
 
-=== "Typescript"
+=== "TypeScript"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<--- "docs/src/ann_indexes.ts:import"
 
-    --8<-- "docs/src/ann_indexes.ts:ingest"
+        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
-    ```
+
+        ```typescript
+        --8<--- "nodejs/examples/ann_indexes.ts:import"
+
+        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
+        ```
+
+    === "vectordb (deprecated)"
+
+        Creating indexes is done via the [lancedb.Table.createIndex](../javascript/interfaces/Table.md/#createIndex) method.
+
+        ```typescript
+        --8<--- "docs/src/ann_indexes.ts:import"
+
+        --8<-- "docs/src/ann_indexes.ts:ingest"
+        ```
 
 === "Rust"
 
@@ -91,27 +105,27 @@ You can specify the GPU device to train IVF partitions via
 
 === "Linux"
 
-     <!-- skip-test -->
+    <!-- skip-test -->
-     ``` { .python .copy }
+    ``` { .python .copy }
-     # Create index using CUDA on Nvidia GPUs.
+    # Create index using CUDA on Nvidia GPUs.
-     tbl.create_index(
+    tbl.create_index(
-          num_partitions=256,
+        num_partitions=256,
-          num_sub_vectors=96,
+        num_sub_vectors=96,
-          accelerator="cuda"
+        accelerator="cuda"
-     )
+    )
-     ```
+    ```
 
 === "MacOS"
 
-     <!-- skip-test -->
+    <!-- skip-test -->
-     ```python
+    ```python
-     # Create index using MPS on Apple Silicon.
+    # Create index using MPS on Apple Silicon.
-     tbl.create_index(
+    tbl.create_index(
-          num_partitions=256,
+        num_partitions=256,
-          num_sub_vectors=96,
+        num_sub_vectors=96,
-          accelerator="mps"
+        accelerator="mps"
-     )
+    )
-     ```
+    ```
 
 Troubleshooting:
 
@@ -150,11 +164,19 @@ There are a couple of parameters that can be used to fine-tune the search:
     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
     ```
 
-=== "Typescript"
+=== "TypeScript"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/ann_indexes.ts:search1"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search1"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/ann_indexes.ts:search1"
+        ```
 
 === "Rust"
 
@@ -172,15 +194,23 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
 
-     ```python
+    ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
-     ```
+    ```
 
-=== "Typescript"
+=== "TypeScript"
 
-     ```javascript
+    === "@lancedb/lancedb"
-     --8<-- "docs/src/ann_indexes.ts:search2"
+
-     ```
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search2"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```javascript
+        --8<-- "docs/src/ann_indexes.ts:search2"
+        ```
 
 ### Projections (select clause)
 
@@ -188,23 +218,31 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-     ```python
+    ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
-     ```
+    ```
 
 
-     ```text
+    ```text
-                                                   vector _distance
+                                                vector _distance
-     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+    0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+    1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-     ...
+    ...
-     ```
+    ```
 
-=== "Typescript"
+=== "TypeScript"
 
-     ```typescript
+    === "@lancedb/lancedb"
-     --8<-- "docs/src/ann_indexes.ts:search3"
+
-     ```
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search3"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/ann_indexes.ts:search3"
+        ```
 
 ## FAQ
 

docs/src/basic.md

@@ -16,12 +16,43 @@
       pip install lancedb
       ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-      ```shell
+        ```shell
-      npm install vectordb
+        npm install @lancedb/lancedb
-      ```
+        ```
+        !!! note "Bundling `@lancedb/lancedb` apps with Webpack"
 
+            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+            ```javascript
+            /** @type {import('next').NextConfig} */
+            module.exports = ({
+            webpack(config) {
+                config.externals.push({ '@lancedb/lancedb': '@lancedb/lancedb' })
+                return config;
+            }
+            })
+            ```
+    === "vectordb (deprecated)"
+
+        ```shell
+        npm install vectordb
+        ```
+        !!! note "Bundling `vectordb` apps with Webpack"
+
+            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+            ```javascript
+            /** @type {import('next').NextConfig} */
+            module.exports = ({
+            webpack(config) {
+                config.externals.push({ vectordb: 'vectordb' })
+                return config;
+            }
+            })
+            ```
 === "Rust"
 
     ```shell
@@ -58,14 +89,21 @@ recommend switching to stable releases.
       pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
       ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-      ```shell
+    === "@lancedb/lancedb"
-      npm install vectordb@preview
+
-      ```
+        ```shell
+        npm install @lancedb/lancedb@preview
+        ```
+    === "vectordb (deprecated)"
+
+        ```shell
+        npm install vectordb@preview
+        ```
 
 === "Rust"
-    
+
     We don't push preview releases to crates.io, but you can referent the tag
     in GitHub within your Cargo dependencies:
 
@@ -93,23 +131,22 @@ recommend switching to stable releases.
         use the same syntax as the asynchronous API.  To help with this migration we
         have created a [migration guide](migration.md) detailing the differences.
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:import"
 
-    --8<-- "docs/src/basic_legacy.ts:open_db"
+        ```typescript
-    ```
+        import * as lancedb from "@lancedb/lancedb";
+        import * as arrow from "apache-arrow";
 
-    !!! note "`@lancedb/lancedb` vs. `vectordb`"
+        --8<-- "nodejs/examples/basic.ts:connect"
+        ```
 
-        The Javascript SDK was originally released as `vectordb`.  In an effort to
+    === "vectordb (deprecated)"
-        reduce maintenance we are aligning our SDKs.  The new, aligned, Javascript
+
-        API is being released as `lancedb`.  If you are starting new work we encourage
+        ```typescript
-        you to try out `lancedb`.  Once the new API is feature complete we will begin
+        --8<-- "docs/src/basic_legacy.ts:open_db"
-        slowly deprecating `vectordb` in favor of `lancedb`. There is a
+        ```
-        [migration guide](migration.md) detailing the differences which will assist
-        you in this process.
 
 === "Rust"
 
@@ -152,15 +189,23 @@ table.
     --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:create_table"
-    ```
 
-    If the table already exists, LanceDB will raise an error by default.
+        ```typescript
-    If you want to overwrite the table, you can pass in `mode="overwrite"`
+        --8<-- "nodejs/examples/basic.ts:create_table"
-    to the `createTable` function.
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_table"
+        ```
+
+        If the table already exists, LanceDB will raise an error by default.
+        If you want to overwrite the table, you can pass in `mode:"overwrite"`
+        to the `createTable` function.
 
 === "Rust"
 
@@ -180,6 +225,9 @@ table.
 
 !!! info "Under the hood, LanceDB reads in the Apache Arrow data and persists it to disk using the [Lance format](https://www.github.com/lancedb/lance)."
 
+!!! info "Automatic embedding generation with Embedding API"
+    When working with embedding models, it is recommended to use the LanceDB embedding API to automatically create vector representation of the data and queries in the background. See the [quickstart example](#using-the-embedding-api) or the embedding API [guide](./embeddings/)
+
 ### Create an empty table
 
 Sometimes you may not have the data to insert into the table at creation time.
@@ -194,11 +242,22 @@ similar to a `CREATE TABLE` statement in SQL.
       --8<-- "python/python/tests/docs/test_basic.py:create_empty_table_async"
       ```
 
-=== "Typescript"
+    !!! note "You can define schema in Pydantic"
+        LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
 
-    ```typescript
+=== "Typescript[^1]"
-    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+
-    ```
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+        ```
 
 === "Rust"
 
@@ -217,11 +276,19 @@ Once created, you can open a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:open_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        const tbl = await db.openTable("myTable");
+        ```
 
-    ```typescript
-    const tbl = await db.openTable("myTable");
-    ```
 
 === "Rust"
 
@@ -238,11 +305,18 @@ If you forget the name of your table, you can always get a listing of all table
     --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
     ```
 
-=== "Javascript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```javascript
+        ```typescript
-    console.log(await db.tableNames());
+        --8<-- "nodejs/examples/basic.ts:table_names"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        console.log(await db.tableNames());
+        ```
 
 === "Rust"
 
@@ -261,11 +335,18 @@ After a table has been created, you can always add more data to it as follows:
     --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```typescript
+        ```typescript
-    --8<-- "docs/src/basic_legacy.ts:add"
+        --8<-- "nodejs/examples/basic.ts:add_data"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:add"
+        ```
 
 === "Rust"
 
@@ -286,11 +367,18 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
 
     This returns a pandas DataFrame with the results.
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```typescript
+        ```typescript
-    --8<-- "docs/src/basic_legacy.ts:search"
+        --8<-- "nodejs/examples/basic.ts:vector_search"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:search"
+        ```
 
 === "Rust"
 
@@ -319,11 +407,18 @@ LanceDB allows you to create an ANN index on a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```{.typescript .ignore}
+        ```typescript
-    --8<-- "docs/src/basic_legacy.ts:create_index"
+        --8<-- "nodejs/examples/basic.ts:create_index"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```{.typescript .ignore}
+        --8<-- "docs/src/basic_legacy.ts:create_index"
+        ```
 
 === "Rust"
 
@@ -351,11 +446,19 @@ This can delete any number of rows that match the filter.
     --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:delete"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:delete_rows"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:delete"
+        ```
 
 === "Rust"
 
@@ -372,9 +475,15 @@ simple or complex as needed. To see what expressions are supported, see the
 
       Read more: [lancedb.table.Table.delete][]
 
-=== "Javascript"
+=== "Typescript[^1]"
 
-      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+    === "@lancedb/lancedb"
+
+        Read more: [lancedb.Table.delete](javascript/interfaces/Table.md#delete)
+
+    === "vectordb (deprecated)"
+
+        Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
 === "Rust"
 
@@ -386,23 +495,31 @@ Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
 
-      ```python
+    ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-      ```
+    ```
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
+    This permanently removes the table and is not recoverable, unlike deleting rows.
-      By default, if the table does not exist an exception is raised. To suppress this,
+    By default, if the table does not exist an exception is raised. To suppress this,
-      you can pass in `ignore_missing=True`.
+    you can pass in `ignore_missing=True`.
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-      ```typescript
+    === "@lancedb/lancedb"
-      --8<-- "docs/src/basic_legacy.ts:drop_table"
-      ```
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
+        ```typescript
-      If the table does not exist an exception is raised.
+        --8<-- "nodejs/examples/basic.ts:drop_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:drop_table"
+        ```
+
+        This permanently removes the table and is not recoverable, unlike deleting rows.
+        If the table does not exist an exception is raised.
 
 === "Rust"
 
@@ -410,22 +527,40 @@ Use the `drop_table()` method on the database to remove a table.
     --8<-- "rust/lancedb/examples/simple.rs:drop_table"
     ```
 
-!!! note "Bundling `vectordb` apps with Webpack"
 
-    If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+## Using the Embedding API
+You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
 
-    ```javascript
+=== "Python"
-    /** @type {import('next').NextConfig} */
+
-    module.exports = ({
+    ```python
-    webpack(config) {
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:imports"
-        config.externals.push({ vectordb: 'vectordb' })
+    --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
-        return config;
-    }
-    })
     ```
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
+        ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/openai.rs:imports"
+    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    ```
+
+Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
+
+
 ## What's next
 
 This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
 
 If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.
+
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/basic_legacy.ts

@@ -24,6 +24,7 @@ const example = async () => {
   );
   // --8<-- [end:create_table]
 
+
   // --8<-- [start:add]
   const newData = Array.from({ length: 500 }, (_, i) => ({
     vector: [i, i + 1],

docs/src/cloud/rest.md

@@ -0,0 +1 @@
+!!swagger ../../openapi.yml!!

docs/src/embeddings/default_embedding_functions.md

@@ -193,13 +193,13 @@ from lancedb.pydantic import LanceModel, Vector
 
 model = get_registry().get("huggingface").create(name='facebook/bart-base')
 
-class TextModel(LanceModel):
+class Words(LanceModel):
     text: str = model.SourceField()
     vector: Vector(model.ndims()) = model.VectorField()
 
 df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
 table = db.create_table("greets", schema=Words)
-table.add()
+table.add(df)
 query = "old greeting"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
@@ -216,7 +216,7 @@ Generate embeddings via the [ollama](https://github.com/ollama/ollama-python) py
 |------------------------|----------------------------|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------|
 | `name`                 | `str`                      | `nomic-embed-text`       | The name of the model.                                                                                                                         |
 | `host`                 | `str`                      | `http://localhost:11434` | The Ollama host to connect to.                                                                                                                 |
-| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the [Modelfile](./modelfile.md#valid-parameters-and-values) such as `temperature`. |
+| `options`              | `ollama.Options` or `dict` | `None`                   | Additional model parameters listed in the documentation for the Modelfile such as `temperature`. |
 | `keep_alive`           | `float` or `str`           | `"5m"`                   | Controls how long the model will stay loaded into memory following the request.                                                                |
 | `ollama_client_kwargs` | `dict`                     | `{}`                     | kwargs that can be past to the `ollama.Client`.                                                                                                |
 
@@ -365,6 +365,107 @@ tbl.add(df)
 rs = tbl.search("hello").limit(1).to_pandas()
 ```
 
+### Cohere Embeddings
+Using cohere API requires cohere package, which can be installed using `pip install cohere`. Cohere embeddings are used to generate embeddings for text data. The embeddings can be used for various tasks like semantic search, clustering, and classification.
+You also need to set the `COHERE_API_KEY` environment variable to use the Cohere API.
+
+Supported models are:
+```
+    * embed-english-v3.0
+    * embed-multilingual-v3.0
+    * embed-english-light-v3.0
+    * embed-multilingual-light-v3.0
+    * embed-english-v2.0
+    * embed-english-light-v2.0
+    * embed-multilingual-v2.0
+```
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"embed-english-v2.0"` | The model ID of the cohere model to use. Supported base models for Text Embeddings: embed-english-v3.0, embed-multilingual-v3.0, embed-english-light-v3.0, embed-multilingual-light-v3.0, embed-english-v2.0, embed-english-light-v2.0, embed-multilingual-v2.0 |
+| `source_input_type` | `str` | `"search_document"` | The type of input data to be used for the source column. |
+| `query_input_type` | `str` | `"search_query"` | The type of input data to be used for the query. |
+
+Cohere supports following input types:
+| Input Type               | Description                          |
+|-------------------------|---------------------------------------|
+| "`search_document`"     | Used for embeddings stored in a vector|
+|                         | database for search use-cases.        |
+| "`search_query`"        | Used for embeddings of search queries |
+|                         | run against a vector DB               |
+| "`semantic_similarity`" | Specifies the given text will be used |
+|                         | for Semantic Textual Similarity (STS) |
+| "`classification`"      | Used for embeddings passed through a  |
+|                         | text classifier.                      |
+| "`clustering`"          | Used for the embeddings run through a |
+|                         | clustering algorithm                  |
+
+Usage Example:
+    
+    ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import EmbeddingFunctionRegistry
+
+        cohere = EmbeddingFunctionRegistry
+            .get_instance()
+            .get("cohere")
+            .create(name="embed-multilingual-v2.0")
+
+        class TextModel(LanceModel):
+            text: str = cohere.SourceField()
+            vector: Vector(cohere.ndims()) =  cohere.VectorField()
+
+        data = [ { "text": "hello world" },
+                { "text": "goodbye world" }]
+
+        db = lancedb.connect("~/.lancedb")
+        tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+        tbl.add(data)
+    ```
+
+### Jina Embeddings
+Jina embeddings are used to generate embeddings for text and image data.
+You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
+
+You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+```python
+    import os
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    os.environ['JINA_API_KEY'] = 'jina_*'
+
+    jina_embed = EmbeddingFunctionRegistry.get_instance().get("jina").create(name="jina-embeddings-v2-base-en")
+
+
+    class TextModel(LanceModel):
+        text: str = jina_embed.SourceField()
+        vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
+
+
+    data = [{"text": "hello world"},
+            {"text": "goodbye world"}]
+
+    db = lancedb.connect("~/.lancedb-2")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+```
+
 ### AWS Bedrock Text Embedding Functions
 AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
 You can do so by using `awscli` and also add your session_token:
@@ -462,7 +563,7 @@ uris = [
 # get each uri as bytes
 image_bytes = [requests.get(uri).content for uri in uris]
 table.add(
-    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
 )
 ```
 Now we can search using text from both the default vector column and the custom vector column
@@ -568,3 +669,54 @@ print(actual.text == "bird")
 ```
 
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).
+
+### Jina Embeddings
+Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
+under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+```python
+    import os
+    import requests
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+    import pandas as pd
+
+    os.environ['JINA_API_KEY'] = 'jina_*'
+
+    db = lancedb.connect("~/.lancedb")
+    func = get_registry().get("jina").create()
+
+
+    class Images(LanceModel):
+        label: str
+        image_uri: str = func.SourceField()  # image uri as the source
+        image_bytes: bytes = func.SourceField()  # image bytes as the source
+        vector: Vector(func.ndims()) = func.VectorField()  # vector column
+        vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
+
+
+    table = db.create_table("images", schema=Images)
+    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+    uris = [
+        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+    ]
+    # get each uri as bytes
+    image_bytes = [requests.get(uri).content for uri in uris]
+    table.add(
+      pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
+    )
+```

docs/src/embeddings/embedding_functions.md

@@ -2,9 +2,12 @@ Representing multi-modal data as vector embeddings is becoming a standard practi
 
 For this purpose, LanceDB introduces an **embedding functions API**, that allow you simply set up once, during the configuration stage of your project. After this, the table remembers it, effectively making the embedding functions *disappear in the background* so you don't have to worry about manually passing callables, and instead, simply focus on the rest of your data engineering pipeline.
 
+!!! Note "LanceDB cloud doesn't support embedding functions yet"
+    LanceDB Cloud does not support embedding functions yet. You need to generate embeddings before ingesting into the table or querying.
+
 !!! warning
-    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself. 
+    Using the embedding function registry means that you don't have to explicitly generate the embeddings yourself.
-    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function 
+    However, if your embedding function changes, you'll have to re-configure your table with the new embedding function
     and regenerate the embeddings. In the future, we plan to support the ability to change the embedding function via
     table metadata and have LanceDB automatically take care of regenerating the embeddings.
 
@@ -13,7 +16,7 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
 === "Python"
     In the LanceDB python SDK, we define a global embedding function registry with
-    many different embedding models and even more coming soon. 
+    many different embedding models and even more coming soon.
     Here's let's an implementation of CLIP as example.
 
     ```python
@@ -23,20 +26,35 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
     clip = registry.get("open-clip").create()
     ```
 
-    You can also define your own embedding function by implementing the `EmbeddingFunction` 
+    You can also define your own embedding function by implementing the `EmbeddingFunction`
     abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
 
-=== "JavaScript""
+=== "TypeScript"
     In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
     embedding function is available.
 
     ```javascript
-    const lancedb = require("vectordb");
+    import * as lancedb from '@lancedb/lancedb'
+    import { getRegistry } from '@lancedb/lancedb/embeddings'
 
     // You need to provide an OpenAI API key
     const apiKey = "sk-..."
     // The embedding function will create embeddings for the 'text' column
-    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+    const func = getRegistry().get("openai").create({apiKey})
+    ```
+=== "Rust"
+    In the Rust SDK, the choices are more limited. For now, only the OpenAI
+    embedding function is available. But unlike the Python and TypeScript SDKs, you need manually register the OpenAI embedding function.
+
+    ```toml
+    // Make sure to include the `openai` feature
+    [dependencies]
+    lancedb = {version = "*", features = ["openai"]}
+    ```
+
+    ```rust
+    --8<-- "rust/lancedb/examples/openai.rs:imports"
+    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
     ```
 
 ## 2. Define the data model or schema
@@ -52,14 +70,14 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
     `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-=== "JavaScript"
+=== "TypeScript"
 
     For the TypeScript SDK, a schema can be inferred from input data, or an explicit
     Arrow schema can be provided.
 
 ## 3. Create table and add data
 
-Now that we have chosen/defined our embedding function and the schema, 
+Now that we have chosen/defined our embedding function and the schema,
 we can create the table and ingest data without needing to explicitly generate
 the embeddings at all:
 
@@ -71,17 +89,26 @@ the embeddings at all:
     table.add([{"image_uri": u} for u in uris])
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const db = await lancedb.connect("data/sample-lancedb");
-    const data = [
-    { text: "pepperoni"},
-    { text: "pineapple"}
-    ]
 
-    const table = await db.createTable("vectors", data, embedding)
+        ```ts
-    ```
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:embedding_function"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const db = await lancedb.connect("data/sample-lancedb");
+        const data = [
+            { text: "pepperoni"},
+            { text: "pineapple"}
+        ]
+
+        const table = await db.createTable("vectors", data, embedding)
+        ```
 
 ## 4. Querying your table
 Not only can you forget about the embeddings during ingestion, you also don't
@@ -94,8 +121,8 @@ need to worry about it when you query the table:
     ```python
     results = (
         table.search("dog")
-        .limit(10)
+            .limit(10)
-        .to_pandas()
+            .to_pandas()
     )
     ```
 
@@ -106,22 +133,32 @@ need to worry about it when you query the table:
     query_image = Image.open(p)
     results = (
         table.search(query_image)
-        .limit(10)
+            .limit(10)
-        .to_pandas()
+            .to_pandas()
     )
     ```
 
     Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
-=== "JavaScript"
+=== "TypeScript"
+
+    === "@lancedb/lancedb"
+
+        ```ts
+        const results = await table.search("What's the best pizza topping?")
+            .limit(10)
+            .toArray()
+        ```
+
+    === "vectordb (deprecated)
+
+        ```ts
+        const results = await table
+            .search("What's the best pizza topping?")
+            .limit(10)
+            .execute()
+        ```
 
-    ```javascript
-    const results = await table
-    .search("What's the best pizza topping?")
-    .limit(10)
-    .execute()
-    ```    
-    
     The above snippet returns an array of records with the top 10 nearest neighbors to the query.
 
 ---

docs/src/embeddings/index.md

@@ -1,13 +1,13 @@
-Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio. 
+Due to the nature of vector embeddings, they can be used to represent any kind of data, from text to images to audio.
-This makes them a very powerful tool for machine learning practitioners. 
+This makes them a very powerful tool for machine learning practitioners.
-However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs 
+However, there's no one-size-fits-all solution for generating embeddings - there are many different libraries and APIs
 (both commercial and open source) that can be used to generate embeddings from structured/unstructured data.
 
 LanceDB supports 3 methods of working with embeddings.
 
 1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
 2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
-3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
+3. You can define your own [custom embedding function](./custom_embedding_function.md)
    that extends the default embedding functions.
 
 For python users, there is also a legacy [with_embeddings API](./legacy.md).
@@ -18,15 +18,103 @@ It is retained for compatibility and will be removed in a future version.
 To get started with embeddings, you can use the built-in embedding functions.
 
 ### OpenAI Embedding function
+
 LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
 
+=== "Python"
+
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = db.create_table("words", schema=Words, mode="overwrite")
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+        )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    --8<--- "nodejs/examples/embedding.ts:imports"
+    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<--- "rust/lancedb/examples/openai.rs:imports"
+    --8<--- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    ```
+
+### Sentence Transformers Embedding function
+LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
+
+=== "Python"
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+    class Words(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    table = db.create_table("words", schema=Words)
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+    )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+
+=== "TypeScript"
+
+    Coming Soon!
+
+=== "Rust"
+
+    Coming Soon!
+
+### Jina Embeddings
+
+LanceDB registers the JinaAI embeddings function in the registry as `jina`. You can pass any supported model name to the `create`. By default it uses `"jina-clip-v1"`.
+`jina-clip-v1` can handle both text and images and other models only support `text`.
+
+You need to pass `JINA_API_KEY` in the environment variable or pass it as `api_key` to `create` method.
+
 ```python
+import os
 import lancedb
 from lancedb.pydantic import LanceModel, Vector
 from lancedb.embeddings import get_registry
+os.environ['JINA_API_KEY'] = "jina_*"
 
 db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
+func = get_registry().get("jina").create(name="jina-clip-v1")
 
 class Words(LanceModel):
     text: str = func.SourceField()
@@ -44,31 +132,3 @@ query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-)
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/fts.md

@@ -2,7 +2,6 @@
 
 LanceDB provides support for full-text search via [Tantivy](https://github.com/quickwit-oss/tantivy) (currently Python only), allowing you to incorporate keyword-based search (based on BM25) in your retrieval solutions. Our goal is to push the FTS integration down to the Rust level in the future, so that it's available for Rust and JavaScript users as well.  Follow along at [this Github issue](https://github.com/lancedb/lance/issues/1195)
 
-A hybrid search solution combining vector and full-text search is also on the way.
 
 ## Installation
 
@@ -55,6 +54,16 @@ This returns the result as a list of dictionaries as follows.
 !!! note
     LanceDB automatically searches on the existing FTS index if the input to the search is of type `str`. If you provide a vector as input, LanceDB will search the ANN index instead.
 
+## Tokenization
+By default the text is tokenized by splitting on punctuation and whitespaces and then removing tokens that are longer than 40 chars. For more language specific tokenization then provide the argument tokenizer_name with the 2 letter language code followed by "_stem". So for english it would be "en_stem".
+
+```python
+table.create_fts_index("text", tokenizer_name="en_stem")
+```
+
+The following [languages](https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html) are currently supported.
+
+
 ## Index multiple columns
 
 If you have multiple string columns to index, there's no need to combine them manually -- simply pass them all as a list to `create_fts_index`:
@@ -140,6 +149,7 @@ is treated as a phrase query.
 In general, a query that's declared as a phrase query will be wrapped in double quotes during parsing, with nested
 double quotes replaced by single quotes.
 
+
 ## Configurations
 
 By default, LanceDB configures a 1GB heap size limit for creating the index. You can

docs/src/guides/storage.md

@@ -32,28 +32,54 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
     db = lancedb.connect("az://bucket/path")
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    AWS S3:
+    === "@lancedb/lancedb"
 
-    ```javascript
+        AWS S3:
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path");
-    ```
 
-    Google Cloud Storage:
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("s3://bucket/path");
+        ```
 
-    ```javascript
+        Google Cloud Storage:
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("gs://bucket/path");
-    ```
 
-    Azure Blob Storage:
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("gs://bucket/path");
+        ```
 
-    ```javascript
+        Azure Blob Storage:
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect("az://bucket/path");
+        ```ts
-    ```
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("az://bucket/path");
+        ```
+
+
+    === "vectordb (deprecated)"
+
+        AWS S3:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path");
+        ```
+
+        Google Cloud Storage:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("gs://bucket/path");
+        ```
+
+        Azure Blob Storage:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("az://bucket/path");
+        ```
 
 In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
 
@@ -78,13 +104,26 @@ If you only want this to apply to one particular connection, you can pass the `s
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect("s3://bucket/path",
+        ```ts
-                                     {storageOptions: {timeout: "60s"}});
+        import * as lancedb from "@lancedb/lancedb";
-    ```
+
+        const db = await lancedb.connect("s3://bucket/path", {
+            storageOptions: {timeout: "60s"}
+        });
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path", {
+            storageOptions: {timeout: "60s"}
+        });
+        ```
 
 Getting even more specific, you can set the `timeout` for only a particular table:
 
@@ -101,18 +140,33 @@ Getting even more specific, you can set the `timeout` for only a particular tabl
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    <!-- skip-test -->
+    === "@lancedb/lancedb"
-    ```javascript
+
-    const lancedb = require("lancedb");
+        <!-- skip-test -->
-    const db = await lancedb.connect("s3://bucket/path");
+        ```ts
-    const table = db.createTable(
+        import * as lancedb from "@lancedb/lancedb";
-        "table",
+        const db = await lancedb.connect("s3://bucket/path");
-        [{ a: 1, b: 2}],
+        const table = db.createTable(
-        {storageOptions: {timeout: "60s"}}
+            "table",
-    );
+            [{ a: 1, b: 2}],
-    ```
+            {storageOptions: {timeout: "60s"}}
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        <!-- skip-test -->
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path");
+        const table = db.createTable(
+            "table",
+            [{ a: 1, b: 2}],
+            {storageOptions: {timeout: "60s"}}
+        );
+        ```
 
 !!! info "Storage option casing"
 
@@ -135,7 +189,6 @@ There are several options that can be set for all object stores, mostly related
 | `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
 | `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
 
-
 ### AWS S3
 
 To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
@@ -155,21 +208,39 @@ These can be set as environment variables or passed in the `storage_options` par
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "s3://bucket/path",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "s3://bucket/path",
-                awsAccessKeyId: "my-access-key",
+            {
-                awsSecretAccessKey: "my-secret-key",
+                storageOptions: {
-                awsSessionToken: "my-session-token",
+                    awsAccessKeyId: "my-access-key",
+                    awsSecretAccessKey: "my-secret-key",
+                    awsSessionToken: "my-session-token",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    awsAccessKeyId: "my-access-key",
+                    awsSecretAccessKey: "my-secret-key",
+                    awsSessionToken: "my-session-token",
+                }
+            }
+        );
+        ```
 
 Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
 
@@ -188,7 +259,6 @@ The following keys can be used as both environment variables or keys in the `sto
 | `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
 | `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
 
-
 !!! tip "Automatic cleanup for failed writes"
 
     LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
@@ -265,6 +335,108 @@ For **read-only access**, LanceDB will need a policy such as:
 }
 ```
 
+#### DynamoDB Commit Store for concurrent writes
+
+By default, S3 does not support concurrent writes. Having two or more processes
+writing to the same table at the same time can lead to data corruption. This is
+because S3, unlike other object stores, does not have any atomic put or copy
+operation.
+
+To enable concurrent writes, you can configure LanceDB to use a DynamoDB table
+as a commit store. This table will be used to coordinate writes between
+different processes. To enable this feature, you must modify your connection
+URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the
+name of the table to use.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+
+    const db = await lancedb.connect(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    );
+    ```
+
+The DynamoDB table must be created with the following schema:
+
+- Hash key: `base_uri` (string)
+- Range key: `version` (number)
+
+You can create this programmatically with:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import boto3
+
+    dynamodb = boto3.client("dynamodb")
+    table = dynamodb.create_table(
+        TableName=table_name,
+        KeySchema=[
+            {"AttributeName": "base_uri", "KeyType": "HASH"},
+            {"AttributeName": "version", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "base_uri", "AttributeType": "S"},
+            {"AttributeName": "version", "AttributeType": "N"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+    )
+    ```
+
+=== "JavaScript"
+
+    <!-- skip-test -->
+    ```javascript
+    import {
+      CreateTableCommand,
+      DynamoDBClient,
+    } from "@aws-sdk/client-dynamodb";
+
+    const dynamodb = new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+    const command = new CreateTableCommand({
+      TableName: table_name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    ```
+
+
 #### S3-compatible stores
 
 LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
@@ -282,20 +454,37 @@ LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you m
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "s3://bucket/path",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "s3://bucket/path",
-                region: "us-east-1",
+            {
-                endpoint: "http://minio:9000",
+                storageOptions: {
+                    region: "us-east-1",
+                    endpoint: "http://minio:9000",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    endpoint: "http://minio:9000",
+                }
+            }
+        );
+        ```
 
 This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
 
@@ -326,21 +515,37 @@ To configure LanceDB to use an S3 Express endpoint, you must set the storage opt
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "s3://my-bucket--use1-az4--x-s3/path",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "s3://my-bucket--use1-az4--x-s3/path",
-                region: "us-east-1",
+            {
-                s3Express: "true",
+                storageOptions: {
+                    region: "us-east-1",
+                    s3Express: "true",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
 
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    s3Express: "true",
+                }
+            }
+        );
+        ```
 
 ### Google Cloud Storage
 
@@ -359,26 +564,40 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "gs://my-bucket/my-database",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "gs://my-bucket/my-database",
-                serviceAccount: "path/to/service-account.json",
+            {
+                storageOptions: {
+                    serviceAccount: "path/to/service-account.json",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
 
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "gs://my-bucket/my-database",
+            {
+                storageOptions: {
+                    serviceAccount: "path/to/service-account.json",
+                }
+            }
+        );
+        ```
 
 !!! info "HTTP/2 support"
 
     By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
 
-
 The following keys can be used as both environment variables or keys in the `storage_options` parameter:
 <!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
 
@@ -388,7 +607,6 @@ The following keys can be used as both environment variables or keys in the `sto
 | ``google_service_account_key``        | The serialized service account key.          |
 | ``google_application_credentials``    | Path to the application credentials.         |
 
-
 ### Azure Blob Storage
 
 Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
@@ -407,20 +625,37 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "az://my-container/my-database",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "az://my-container/my-database",
-                accountName: "some-account",
+            {
-                accountKey: "some-key",
+                storageOptions: {
+                    accountName: "some-account",
+                    accountKey: "some-key",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "az://my-container/my-database",
+            {
+                storageOptions: {
+                    accountName: "some-account",
+                    accountKey: "some-key",
+                }
+            }
+        );
+        ```
 
 These keys can be used as both environment variables or keys in the `storage_options` parameter:
 
@@ -445,4 +680,4 @@ These keys can be used as both environment variables or keys in the `storage_opt
 | ``azure_use_azure_cli``               | Use azure cli for acquiring access token.                                                        |
 | ``azure_disable_tagging``             | Disables tagging objects. This can be desirable if not supported by the backing store.           |
 
-<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->
+<!-- TODO: demonstrate how to configure networked file systems for optimal performance -->

docs/src/guides/tables.md

@@ -3,32 +3,45 @@
 
 A Table is a collection of Records in a LanceDB Database. Tables in Lance have a schema that defines the columns and their types. These schemas can include nested columns and can evolve over time.
 
-This guide will show how to create tables, insert data into them, and update the data.  
+This guide will show how to create tables, insert data into them, and update the data.
 
 
 ## Creating a LanceDB Table
 
+Initialize a LanceDB connection and create a table
+
 === "Python"
-    Initialize a LanceDB connection and create a table using one of the many methods listed below.
 
     ```python
     import lancedb
     db = lancedb.connect("./.lancedb")
     ```
 
-=== "Javascript"
-
-    Initialize a VectorDB connection and create a table using one of the many methods listed below.
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        import * as lancedb from "@lancedb/lancedb";
+        import * as arrow from "apache-arrow";
+
+        const uri = "data/sample-lancedb";
+        const db = await lancedb.connect(uri);
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        const lancedb = require("vectordb");
+
+        const uri = "data/sample-lancedb";
+        const db = await lancedb.connect(uri);
+        ```
+
+
+
 ### From list of tuples or dictionaries
 
 === "Python"
@@ -45,102 +58,133 @@ This guide will show how to create tables, insert data into them, and update the
 
     db["my_table"].head()
     ```
+
     !!! info "Note"
-        If the table already exists, LanceDB will raise an error by default. 
+        If the table already exists, LanceDB will raise an error by default.
 
         `create_table` supports an optional `exist_ok` parameter. When set to True
         and the table exists, then it simply opens the existing table. The data you
         passed in will NOT be appended to the table in that case.
 
-        ```python
+    ```python
-        db.create_table("name", data, exist_ok=True)
+    db.create_table("name", data, exist_ok=True)
-        ```
-        
-        Sometimes you want to make sure that you start fresh. If you want to 
-        overwrite the table, you can pass in mode="overwrite" to the createTable function.
-
-        ```python
-        db.create_table("name", data, mode="overwrite")
-        ```
-
-=== "Javascript"
-    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
-
-    ```javascript
-    const tb = await db.createTable("my_table", [{
-        "vector": [3.1, 4.1],
-        "item": "foo",
-        "price": 10.0
-    }, {
-        "vector": [5.9, 26.5],
-        "item": "bar",
-        "price": 20.0
-    }]);
-    ```
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
-
-    ```javascript
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
     ```
 
-    ### From a Pandas DataFrame
+    Sometimes you want to make sure that you start fresh. If you want to
+    overwrite the table, you can pass in mode="overwrite" to the createTable function.
 
     ```python
-    import pandas as pd
+    db.create_table("name", data, mode="overwrite")
-
-    data = pd.DataFrame({
-        "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
-        "lat": [45.5, 40.1],
-        "long": [-122.7, -74.1]
-    })
-
-    db.create_table("my_table", data)
-
-    db["my_table"].head()
     ```
-    !!! info "Note"
+
+=== "Typescript[^1]"
+    You can create a LanceDB table in JavaScript using an array of records as follows.
+
+    === "@lancedb/lancedb"
+
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table"
+        ```
+
+        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use `apache-arrow` to declare a schema
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_with_schema"
+        ```
+
+        !!! info "Note"
+            `createTable` supports an optional `existsOk` parameter. When set to true
+            and the table exists, then it simply opens the existing table. The data you
+            passed in will NOT be appended to the table in that case.
+
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_exists_ok"
+        ```
+
+        Sometimes you want to make sure that you start fresh. If you want to
+        overwrite the table, you can pass in mode: "overwrite" to the createTable function.
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_overwrite"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/basic_legacy.ts:create_table"
+        ```
+
+        !!! warning
+            `existsOk` option is not supported in `vectordb`
+
+        Sometimes you want to make sure that you start fresh. If you want to
+        overwrite the table, you can pass in mode: "overwrite" to the createTable function.
+
+        ```ts
+        const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
+        ```
+
+### From a Pandas DataFrame
+
+```python
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+    "lat": [45.5, 40.1],
+    "long": [-122.7, -74.1]
+})
+
+db.create_table("my_table", data)
+
+db["my_table"].head()
+```
+
+!!! info "Note"
     Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
 
-    The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. 
+The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type.
 
-    ```python
+```python
-    custom_schema = pa.schema([
+custom_schema = pa.schema([
-    pa.field("vector", pa.list_(pa.float32(), 4)),
+pa.field("vector", pa.list_(pa.float32(), 4)),
-    pa.field("lat", pa.float32()),
+pa.field("lat", pa.float32()),
-    pa.field("long", pa.float32())
+pa.field("long", pa.float32())
-    ])
+])
 
-    table = db.create_table("my_table", data, schema=custom_schema)
+table = db.create_table("my_table", data, schema=custom_schema)
-    ```
+```
 
 ### From a Polars DataFrame
 
-    LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
+LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
-    written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
+written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
-    under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
+under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
-    is on the way.
+is on the way.
 
-    ```python
+```python
-    import polars as pl
+import polars as pl
 
-    data = pl.DataFrame({
+data = pl.DataFrame({
-        "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
-        "item": ["foo", "bar"],
+    "item": ["foo", "bar"],
-        "price": [10.0, 20.0]
+    "price": [10.0, 20.0]
-    })
+})
-    table = db.create_table("pl_table", data=data)
+table = db.create_table("pl_table", data=data)
-    ```
+```
 
 ### From an Arrow Table
+You can also create LanceDB tables directly from Arrow tables.
+LanceDB supports float16 data type!
+
 === "Python"
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports float16 data type!
 
     ```python
     import pyarrows as pa
     import numpy as np
-    
+
     dim = 16
     total = 2
     schema = pa.schema(
@@ -160,13 +204,19 @@ This guide will show how to create tables, insert data into them, and update the
     tbl = db.create_table("f16_tbl", data, schema=schema)
     ```
 
-=== "Javascript"
+=== "Typescript[^1]"
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports Float16 data type!
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:create_f16_table"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_f16_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_f16_table"
+        ```
 
 ### From Pydantic Models
 
@@ -225,7 +275,7 @@ class NestedSchema(LanceModel):
 tbl = db.create_table("nested_table", schema=NestedSchema, mode="overwrite")
 ```
 
-This creates a struct column called "document" that has two subfields 
+This creates a struct column called "document" that has two subfields
 called "content" and "source":
 
 ```
@@ -236,7 +286,7 @@ vector: fixed_size_list<item: float>[1536] not null
     child 0, item: float
 document: struct<content: string not null, source: string not null> not null
     child 0, content: string not null
-    child 1, source: string not null    
+    child 1, source: string not null
 ```
 
 #### Validators
@@ -261,7 +311,7 @@ class TestModel(LanceModel):
     @classmethod
     def tz_must_match(cls, dt: datetime) -> datetime:
         assert dt.tzinfo == tz
-        return dt        
+        return dt
 
 ok = TestModel(dt_with_tz=datetime.now(tz))
 
@@ -329,23 +379,24 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
     tbl = db.open_table("my_table")
     ```
 
-=== "JavaScript"
+=== "Typescript[^1]"
+
     If you forget the name of your table, you can always get a listing of all table names.
 
-    ```javascript
+    ```typescript
     console.log(await db.tableNames());
     ```
 
     Then, you can open any existing tables.
 
-    ```javascript
+    ```typescript
     const tbl = await db.openTable("my_table");
     ```
 
 ## Creating empty table
+You can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
 
 === "Python"
-    In Python, you can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
 
     ```python
 
@@ -364,8 +415,8 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
     tbl = db.create_table("empty_table_add", schema=schema)
     ```
 
-    Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not 
+    Alternatively, you can also use Pydantic to specify the schema for the empty table. Note that we do not
-    directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel` 
+    directly import `pydantic` but instead use `lancedb.pydantic` which is a subclass of `pydantic.BaseModel`
     that has been extended to support LanceDB specific types like `Vector`.
 
     ```python
@@ -382,9 +433,23 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
 
     Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+        ```
+
 ## Adding to a table
 
-After a table has been created, you can always add more data to it using the various methods available.
+After a table has been created, you can always add more data to it usind the `add` method
 
 === "Python"
     You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or `Iterator[pa.RecordBatch]`. Below are some examples.
@@ -452,8 +517,27 @@ After a table has been created, you can always add more data to it using the var
     tbl.add(pydantic_model_items)
     ```
 
+    ??? "Ingesting Pydantic models with LanceDB embedding API"
+        When using LanceDB's embedding API, you can add Pydantic models directly to the table. LanceDB will automatically convert the `vector` field to a vector before adding it to the table. You need to specify the default value of `vector` feild as None to allow LanceDB to automatically vectorize the data.
 
-=== "JavaScript"
+        ```python
+        import lancedb
+        from lancedb.pydantic import LanceModel, Vector
+        from lancedb.embeddings import get_registry
+
+        db = lancedb.connect("~/tmp")
+        embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.5")
+
+        class Schema(LanceModel):
+            text: str = embed_fcn.SourceField()
+            vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField(default=None)
+
+        tbl = db.create_table("my_table", schema=Schema, mode="overwrite")
+        models = [Schema(text="hello"), Schema(text="world")]
+        tbl.add(models)
+        ```
+
+=== "Typescript[^1]"
 
     ```javascript
     await tbl.add(
@@ -509,15 +593,15 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     # 0  3  [5.0, 6.0]
     ```
 
-=== "JavaScript"
+=== "Typescript[^1]"
 
-    ```javascript
+    ```ts
     await tbl.delete('item = "fizz"')
     ```
 
     ### Deleting row with specific column value
 
-    ```javascript
+    ```ts
     const con = await lancedb.connect("./.lancedb")
     const data = [
       {id: 1, vector: [1, 2]},
@@ -531,7 +615,7 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
 
     ### Delete from a list of values
 
-    ```javascript
+    ```ts
     const to_remove = [1, 5];
     await tbl.delete(`id IN (${to_remove.join(",")})`)
     await tbl.countRows() // Returns 1
@@ -588,26 +672,49 @@ This can be used to update zero to all rows depending on how many rows match the
     2  2  [10.0, 10.0]
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
-    API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
+    === "@lancedb/lancedb"
 
-    ```javascript
+        API Reference: [lancedb.Table.update](../js/classes/Table.md/#update)
-    const lancedb = require("vectordb");
 
-    const db = await lancedb.connect("./.lancedb");
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
 
-    const data = [
+        const db = await lancedb.connect("./.lancedb");
-      {x: 1, vector: [1, 2]},
-      {x: 2, vector: [3, 4]},
-      {x: 3, vector: [5, 6]},
-    ];
-    const tbl = await db.createTable("my_table", data)
 
-    await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        const data = [
-    ```
+            {x: 1, vector: [1, 2]},
+            {x: 2, vector: [3, 4]},
+            {x: 3, vector: [5, 6]},
+        ];
+        const tbl = await db.createTable("my_table", data)
 
-The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
+        await tbl.update({vector: [10, 10]}, { where: "x = 2"})
+        ```
+
+    === "vectordb (deprecated)"
+
+        API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
+
+        ```ts
+        const lancedb = require("vectordb");
+
+        const db = await lancedb.connect("./.lancedb");
+
+        const data = [
+            {x: 1, vector: [1, 2]},
+            {x: 2, vector: [3, 4]},
+            {x: 3, vector: [5, 6]},
+        ];
+        const tbl = await db.createTable("my_table", data)
+
+        await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        ```
+
+#### Updating using a sql query
+
+  The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
 
 === "Python"
 
@@ -626,16 +733,47 @@ The `values` parameter is used to provide the new values for the columns as lite
     2  3  [10.0, 10.0]
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    await tbl.update({ valuesSql: { x: "x + 1" } })
+
-    ```
+        Coming Soon!
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        await tbl.update({ valuesSql: { x: "x + 1" } })
+        ```
 
 !!! info "Note"
 
     When rows are updated, they are moved out of the index. The row will still show up in ANN queries, but the query will not be as fast as it would be if the row was in the index. If you update a large proportion of rows, consider rebuilding the index afterwards.
 
+## Drop a table
+
+Use the `drop_table()` method on the database to remove a table.
+
+=== "Python"
+
+      ```python
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      By default, if the table does not exist an exception is raised. To suppress this,
+      you can pass in `ignore_missing=True`.
+
+=== "TypeScript"
+
+      ```typescript
+      --8<-- "docs/src/basic_legacy.ts:drop_table"
+      ```
+
+      This permanently removes the table and is not recoverable, unlike deleting rows.
+      If the table does not exist an exception is raised.
+
+
 ## Consistency
 
 In LanceDB OSS, users can set the `read_consistency_interval` parameter on connections to achieve different levels of read consistency. This parameter determines how frequently the database synchronizes with the underlying storage system to check for updates made by other processes. If another process updates a table, the database will not see the changes until the next synchronization.
@@ -651,7 +789,7 @@ There are three possible settings for `read_consistency_interval`:
     This is only tune-able in LanceDB OSS. In LanceDB Cloud, readers are always eventually consistent.
 
 === "Python"
-    
+
     To set strong consistency, use `timedelta(0)`:
 
     ```python
@@ -673,33 +811,35 @@ There are three possible settings for `read_consistency_interval`:
     ```python
     db = lancedb.connect("./.lancedb")
     table = db.open_table("my_table")
-    
+
     # (Other writes happen to my_table from another process)
 
     # Check for updates
     table.checkout_latest()
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
     To set strong consistency, use `0`:
 
-    ```javascript
+    ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
     const table = await db.openTable("my_table");
     ```
 
     For eventual consistency, specify the update interval as seconds:
 
-    ```javascript
+    ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
     const table = await db.openTable("my_table");
     ```
 
-<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007 
+<!-- Node doesn't yet support the version time travel: https://github.com/lancedb/lancedb/issues/1007
     Once it does, we can show manual consistency check for Node as well.
 -->
 
 ## What's next?
 
-Learn the best practices on creating an ANN index and getting the most out of it.
+Learn the best practices on creating an ANN index and getting the most out of it.
+
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -0,0 +1,131 @@
+## Improving retriever performance
+
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
+
+There are serveral ways to improve the performance of retrievers. Some of the common techniques are:
+
+* Using different query types
+* Using hybrid search
+* Fine-tuning the embedding models
+* Using different embedding models
+
+Using different embedding models is something that's very specific to the use case and the data. So we will not discuss it here. In this section, we will discuss the first three techniques.
+
+
+!!! note "Note"
+    We'll be using a simple metric called "hit-rate" for evaluating the performance of the retriever across this guide. Hit-rate is the percentage of queries for which the retriever returned the correct answer in the top-k results. For example, if the retriever returned the correct answer in the top-3 results for 70% of the queries, then the hit-rate@3 is 0.7.
+
+
+## The dataset
+We'll be using a QA dataset generated using a LLama2 review paper. The dataset contains 221 query, context and answer triplets. The queries and answers are generated using GPT-4 based on a given query. Full script used to generate the dataset can be found on this [repo](https://github.com/lancedb/ragged). It can be downloaded from [here](https://github.com/AyushExel/assets/blob/main/data_qa.csv)
+
+### Using different query types
+Let's setup the embeddings and the dataset first. We'll use the LanceDB's `huggingface` embeddings integration for this guide. 
+
+```python
+import lancedb
+import pandas as pd
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import Vector, LanceModel
+
+db = lancedb.connect("~/lancedb/query_types")
+df = pd.read_csv("data_qa.csv")
+
+embed_fcn = get_registry().get("huggingface").create(name="BAAI/bge-small-en-v1.")
+
+class Schema(LanceModel):
+    context: str = embed_fcn.SourceField()
+    vector: Vector(embed_fcn.ndims()) = embed_fcn.VectorField()
+
+table = db.create_table("qa", schema=Schema)
+table.add(df[["context"]].to_dict(orient="records"))
+
+queries = df["query"].tolist()
+```
+
+Now that we have the dataset and embeddings table set up, here's how you can run different query types on the dataset.
+
+* <b> Vector Search: </b>
+
+    ```python
+    table.search(quries[0], query_type="vector").limit(5).to_pandas()
+    ```
+    By default, LanceDB uses vector search query type for searching and it automatically converts the input query to a vector before searching when using embedding API. So, the following statement is equivalent to the above statement.
+
+    ```python
+    table.search(quries[0]).limit(5).to_pandas()
+    ```
+
+    Vector or semantic search is useful when you want to find documents that are similar to the query in terms of meaning.
+
+---
+
+* <b> Full-text Search: </b>
+    
+    FTS requires creating an index on the column you want to search on. `replace=True` will replace the existing index if it exists.
+    Once the index is created, you can search using the `fts` query type.
+    ```python
+    table.create_fts_index("context", replace=True)
+    table.search(quries[0], query_type="fts").limit(5).to_pandas()
+    ```
+
+    Full-text search is useful when you want to find documents that contain the query terms.
+
+---
+
+* <b> Hybrid Search: </b>
+
+    Hybrid search is a combination of vector and full-text search. Here's how you can run a hybrid search query on the dataset.
+    ```python
+    table.search(quries[0], query_type="hybrid").limit(5).to_pandas()
+    ```
+    Hybrid search requires a reranker to combine and rank the results from vector and full-text search. We'll cover reranking as a concept in the next section.
+
+    Hybrid search is useful when you want to combine the benefits of both vector and full-text search.
+
+    !!! note "Note"
+        By default, it uses `LinearCombinationReranker` that combines the scores from vector and full-text search using a weighted linear combination. It is the simplest reranker implementation available in LanceDB. You can also use other rerankers like `CrossEncoderReranker` or `CohereReranker` for reranking the results.
+        Learn more about rerankers [here](https://lancedb.github.io/lancedb/reranking/)
+
+    
+
+### Hit rate evaluation results
+
+Now that we have seen how to run different query types on the dataset, let's evaluate the hit-rate of each query type on the dataset.
+For brevity, the entire evaluation script is not shown here. You can find the complete evaluation and benchmarking utility scripts [here](https://github.com/lancedb/ragged).
+
+Here are the hit-rate results for the dataset:
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.640 |
+| Full-text Search | 0.595 |
+| Hybrid Search (w/ LinearCombinationReranker) | 0.645 |
+
+**Choosing query type** is very specific to the use case and the data. This synthetic dataset has been generated to be semantically challenging, i.e, the queries don't have a lot of keywords in common with the context. So, vector search performs better than full-text search. However, in real-world scenarios, full-text search might perform better than vector search. Hybrid search is a good choice when you want to combine the benefits of both vector and full-text search.
+
+### Evaluation results on other datasets
+
+The hit-rate results can vary based on the dataset and the query type. Here are the hit-rate results for the other datasets using the same embedding function.
+
+* <b> SQuAD Dataset: </b>
+
+    | Query Type | Hit-rate@5 |
+    | --- | --- |
+    | Vector Search | 0.822 |
+    | Full-text Search | 0.835 |
+    | Hybrid Search (w/ LinearCombinationReranker) | 0.8874 |
+
+* <b> Uber10K sec filing Dataset: </b>
+
+    | Query Type | Hit-rate@5 |
+    | --- | --- |
+    | Vector Search | 0.608 |
+    | Full-text Search | 0.82 |
+    | Hybrid Search (w/ LinearCombinationReranker) | 0.80 |
+
+In these standard datasets, FTS seems to perform much better than vector search because the queries have a lot of keywords in common with the context. So, in general choosing the query type is very specific to the use case and the data.
+
+

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -0,0 +1,80 @@
+Continuing from the previous section, we can now rerank the results using more complex rerankers.
+
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+## Reranking search results
+You can rerank any search results using a reranker. The syntax for reranking is as follows:
+
+```python
+from lancedb.rerankers import LinearCombinationReranker
+
+reranker = LinearCombinationReranker()
+table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
+```
+Based on the `query_type`, the `rerank()` function can accept other arguments as well. For example, hybrid search accepts a `normalize` param to determine the score normalization method.
+
+!!! note "Note"
+    LanceDB provides a `Reranker` base class that can be extended to implement custom rerankers. Each reranker must implement the `rerank_hybrid` method. `rerank_vector` and `rerank_fts` methods are optional. For example, the `LinearCombinationReranker` only implements the `rerank_hybrid` method and so it can only be used for reranking hybrid search results.
+
+## Choosing a Reranker
+There are many rerankers available in LanceDB like `CrossEncoderReranker`, `CohereReranker`, and `ColBERT`. The choice of reranker depends on the dataset and the application. You can even implement you own custom reranker by extending the `Reranker` class. For more details about each available reranker and performance comparison, refer to the [rerankers](https://lancedb.github.io/lancedb/reranking/) documentation.
+
+In this example, we'll use the `CohereReranker` to rerank the search results. It requires  `cohere` to be installed and `COHERE_API_KEY` to be set in the environment. To get your API key, sign up on [Cohere](https://cohere.ai/).
+
+```python
+from lancedb.rerankers import CohereReranker
+
+# use Cohere reranker v3
+reranker = CohereReranker(model_name="rerank-english-v3.0") # default model is "rerank-english-v2.0"
+```
+
+### Reranking search results
+Now we can rerank all query type results using the `CohereReranker`:
+
+```python
+
+# rerank hybrid search results
+table.search(quries[0], query_type="hybrid").rerank(reranker=reranker).limit(5).to_pandas()
+
+# rerank vector search results
+table.search(quries[0], query_type="vector").rerank(reranker=reranker).limit(5).to_pandas()
+
+# rerank fts search results
+table.search(quries[0], query_type="fts").rerank(reranker=reranker).limit(5).to_pandas()
+```
+
+Each reranker can accept additional arguments. For example, `CohereReranker` accepts `top_k` and `batch_size` params to control the number of documents to rerank and the batch size for reranking respectively. Similarly, a custom reranker can accept any number of arguments based on the implementation. For example, a reranker can accept a `filter` that implements some custom logic to filter out documents before reranking.
+
+## Results
+
+Let us take a look at the same datasets from the previous sections, using the same embedding table but with Cohere reranker applied to all query types.
+
+!!! note "Note"
+    When reranking fts or vector search results, the search results are over-fetched by a factor of 2 and then reranked. From the reranked set, `top_k` (5 in this case) results are taken. This is done because reranking will have no effect on the hit-rate if we only fetch the `top_k` results.
+
+### Synthetic LLama2 paper dataset
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector |  0.640 |
+| FTS   |  0.595  |
+| Reranked vector | 0.677    |
+| Reranked fts  | 0.672    |
+| Hybrid | 0.759 |
+
+### SQuAD Dataset
+
+
+### Uber10K sec filing Dataset
+
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector |  0.608 |
+| FTS   |  0.824  |
+| Reranked vector | 0.671    |
+| Reranked fts  | 0.843    |
+| Hybrid | 0.849 |
+
+
+
+

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -0,0 +1,82 @@
+## Finetuning the Embedding Model
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
+
+We'll use the same dataset as in the previous sections. Start off by splitting the dataset into training and validation sets:
+```python
+from sklearn.model_selection import train_test_split
+
+train_df, validation_df = train_test_split("data_qa.csv", test_size=0.2, random_state=42)
+
+train_df.to_csv("data_train.csv", index=False)
+validation_df.to_csv("data_val.csv", index=False)
+```
+
+You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
+
+
+Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
+```python
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.readers.file import PagedCSVReader
+from llama_index.finetuning import generate_qa_embedding_pairs
+from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
+
+def load_corpus(file):
+    loader = PagedCSVReader(encoding="utf-8")
+    docs = loader.load_data(file=Path(file))
+
+    parser = SentenceSplitter()
+    nodes = parser.get_nodes_from_documents(docs)
+
+    return nodes
+
+from llama_index.llms.openai import OpenAI
+
+
+train_dataset = generate_qa_embedding_pairs(
+    llm=OpenAI(model="gpt-3.5-turbo"), nodes=train_nodes, verbose=False
+)
+val_dataset = generate_qa_embedding_pairs(
+    llm=OpenAI(model="gpt-3.5-turbo"), nodes=val_nodes, verbose=False
+)
+```
+
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 
+
+```python
+from llama_index.finetuning import SentenceTransformersFinetuneEngine
+
+finetune_engine = SentenceTransformersFinetuneEngine(
+    train_dataset,
+    model_id="BAAI/bge-small-en-v1.5",
+    model_output_path="tuned_model",
+    val_dataset=val_dataset,
+)
+finetune_engine.finetune()
+embed_model = finetune_engine.get_finetuned_model()
+```
+This saves the fine tuned embedding model in `tuned_model` folder. This al
+
+# Evaluation results
+In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 
+On performing the same hit-rate evaluation as before, we see a significant improvement in the hit-rate across all query types.
+
+### Baseline
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.640 |
+| Full-text Search | 0.595 |
+| Reranked Vector Search | 0.677 |
+| Reranked Full-text Search | 0.672 |
+| Hybrid Search (w/ CohereReranker) | 0.759|
+
+### Fine-tuned model ( 2 iterations )
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.672 |
+| Full-text Search | 0.595 |
+| Reranked Vector Search | 0.754 |
+| Reranked Full-text Search | 0.672|
+| Hybrid Search (w/ CohereReranker) | 0.768 |

docs/src/hybrid_search/eval.md

@@ -5,7 +5,9 @@ Hybrid Search is a broad (often misused) term. It can mean anything from combini
 ## The challenge of (re)ranking search results
 Once you have a group of the most relevant search results from multiple search sources, you'd likely standardize the score and rank them accordingly. This process can also be seen as another independent step - reranking.
 There are two approaches for reranking search results from multiple sources.
+
 * <b>Score-based</b>: Calculate final relevance scores based on a weighted linear combination of individual search algorithm scores. Example - Weighted linear combination of semantic search & keyword-based search results.
+
 * <b>Relevance-based</b>: Discards the existing scores and calculates the relevance of each search result - query pair. Example - Cross Encoder models
 
 Even though there are many strategies for reranking search results, none works for all cases. Moreover, evaluating them itself is a challenge. Also, reranking can be dataset, application specific so it's hard to generalize.

docs/src/integrations/langchain.md

@@ -2,7 +2,7 @@
 ![Illustration](../assets/langchain.png)
 
 ## Quick Start
-You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model.
+You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. Checkout Complete example here - [LangChain demo](../notebooks/langchain_example.ipynb)
 ```python
 import os
 from langchain.document_loaders import TextLoader
@@ -38,6 +38,8 @@ The exhaustive list of parameters for `LanceDB` vector store are :
 - `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
 - `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
 - `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
+- `reranker`: (Optional) The reranker to use for LanceDB.
+- `relevance_score_fn`: (Optional[Callable[[float], float]]) Langchain relevance score function to be used. Defaults to `None`. 
 
 ```python
 db_url = "db://lang_test" # url of db you created
@@ -54,12 +56,14 @@ vector_store = LanceDB(
 ```
 
 ### Methods 
-To add texts and store respective embeddings automatically:   
+
 ##### add_texts()
 - `texts`: `Iterable` of strings to add to the vectorstore.
 - `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
 - `ids`: Optional `list` of ids to associate with the texts. 
+- `kwargs`: `Any`
 
+This method adds texts and stores respective embeddings automatically.
 
 ```python
 vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
@@ -74,7 +78,6 @@ pd_df.to_csv("docsearch.csv", index=False)
 # you can also create a new vector store object using an older connection object:
 vector_store = LanceDB(connection=tbl, embedding=embeddings)
 ```
-For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
 ##### create_index() 
 - `col_name`: `Optional[str] = None`
 - `vector_col`: `Optional[str] = None`
@@ -82,6 +85,8 @@ For index creation make sure your table has enough data in it. An ANN index is u
 - `num_sub_vectors`: `Optional[int] = 96`
 - `index_cache_size`: `Optional[int] = None`
 
+This method creates an index for the vector store. For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+
 ```python
 # for creating vector index
 vector_store.create_index(vector_col='vector', metric = 'cosine')
@@ -89,4 +94,108 @@ vector_store.create_index(vector_col='vector', metric = 'cosine')
 # for creating scalar index(for non-vector columns)
 vector_store.create_index(col_name='text')
 
-```
+```
+
+##### similarity_search()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `fts`: `Optional[bool] = False`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Return documents most similar to the query without relevance scores
+
+```python
+docs = docsearch.similarity_search(query)
+print(docs[0].page_content)
+```
+
+##### similarity_search_by_vector()
+- `embedding`: `List[float]`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Returns documents most similar to the query vector.
+
+```python
+docs = docsearch.similarity_search_by_vector(query)
+print(docs[0].page_content)
+```
+
+##### similarity_search_with_score()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `kwargs`: `Any`
+
+Returns documents most similar to the query string with relevance scores, gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
+
+```python
+docs = docsearch.similarity_search_with_relevance_scores(query)
+print("relevance score - ", docs[0][1])
+print("text- ", docs[0][0].page_content[:1000])
+```
+
+##### similarity_search_by_vector_with_relevance_scores()
+- `embedding`: `List[float]`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Return documents most similar to the query vector with relevance scores.
+Relevance score 
+
+```python
+docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)
+print("relevance score - ", docs[0][1])
+print("text- ", docs[0][0].page_content[:1000])
+```
+
+##### max_marginal_relevance_search()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `fetch_k` : Number of Documents to fetch to pass to MMR algorithm, `Optional[int] = None`
+- `lambda_mult`: Number between 0 and 1 that determines the degree
+                        of diversity among the results with 0 corresponding
+                        to maximum diversity and 1 to minimum diversity.
+                        Defaults to 0.5. `float = 0.5`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `kwargs`: `Any`
+
+Returns docs selected using the maximal marginal relevance(MMR).
+Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.
+
+Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 
+
+```python
+result = docsearch.max_marginal_relevance_search(
+        query="text"
+    )
+result_texts = [doc.page_content for doc in result]
+print(result_texts)
+
+## search by vector :
+result = docsearch.max_marginal_relevance_search_by_vector(
+        embeddings.embed_query("text")
+    )
+result_texts = [doc.page_content for doc in result]
+print(result_texts)
+```
+
+##### add_images()
+- `uris` : File path to the image. `List[str]`.
+- `metadatas` : Optional list of metadatas. `(Optional[List[dict]], optional)`
+- `ids` : Optional list of IDs. `(Optional[List[str]], optional)`
+
+Adds images by automatically creating their embeddings and adds them to the vectorstore.
+
+```python
+vec_store.add_images(uris=image_uris) 
+# here image_uris are local fs paths to the images.
+```
+
+

docs/src/integrations/llamaIndex.md

@@ -0,0 +1,142 @@
+# Llama-Index
+![Illustration](../assets/llama-index.jpg)
+
+## Quick start
+You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. 
+You can run the below script to try it out :
+```python
+import logging
+import sys
+
+# Uncomment to see debug logs
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
+
+from llama_index.core import SimpleDirectoryReader, Document, StorageContext
+from llama_index.core import VectorStoreIndex
+from llama_index.vector_stores.lancedb import LanceDBVectorStore
+import textwrap
+import openai
+
+openai.api_key = "sk-..."
+
+documents = SimpleDirectoryReader("./data/your-data-dir/").load_data()
+print("Document ID:", documents[0].doc_id, "Document Hash:", documents[0].hash)
+
+## For LanceDB cloud :
+# vector_store = LanceDBVectorStore( 
+#     uri="db://db_name", # your remote DB URI
+#     api_key="sk_..", # lancedb cloud api key
+#     region="your-region" # the region you configured
+#     ...
+# )
+
+vector_store = LanceDBVectorStore(
+    uri="./lancedb", mode="overwrite", query_type="vector"
+)
+storage_context = StorageContext.from_defaults(vector_store=vector_store)
+
+index = VectorStoreIndex.from_documents(
+    documents, storage_context=storage_context
+)
+lance_filter = "metadata.file_name = 'paul_graham_essay.txt' "
+retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter})
+response = retriever.retrieve("What did the author do growing up?")
+```
+
+Checkout Complete example here - [LlamaIndex demo](../notebooks/LlamaIndex_example.ipynb)
+
+### Filtering
+For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
+```python
+from llama_index.core.vector_stores import (
+    MetadataFilters,
+    FilterOperator,
+    FilterCondition,
+    MetadataFilter,
+)
+
+query_filters = MetadataFilters(
+    filters=[
+        MetadataFilter(
+            key="creation_date", operator=FilterOperator.EQ, value="2024-05-23"
+        ),
+        MetadataFilter(
+            key="file_size", value=75040, operator=FilterOperator.GT
+        ),
+    ],
+    condition=FilterCondition.AND,
+)
+```
+
+### Hybrid Search
+For complete documentation, refer [here](https://lancedb.github.io/lancedb/hybrid_search/hybrid_search/). This example uses the `colbert` reranker. Make sure to install necessary dependencies for the reranker you choose.
+```python
+from lancedb.rerankers import ColbertReranker
+
+reranker = ColbertReranker()
+vector_store._add_reranker(reranker)
+
+query_engine = index.as_query_engine(
+    filters=query_filters,
+    vector_store_kwargs={
+        "query_type": "hybrid",
+    }
+)
+
+response = query_engine.query("How much did Viaweb charge per month?")
+```
+
+In the above snippet, you can change/specify query_type again when creating the engine/retriever.
+
+## API reference
+The exhaustive list of parameters for `LanceDBVectorStore` vector store are :  
+- `connection`: Optional, `lancedb.db.LanceDBConnection` connection object to use. If not provided, a new connection will be created.
+- `uri`: Optional[str], the uri of your database. Defaults to `"/tmp/lancedb"`.
+- `table_name` : Optional[str], Name of your table in the database. Defaults to `"vectors"`.
+- `table`: Optional[Any], `lancedb.db.LanceTable` object to be passed. Defaults to `None`. 
+- `vector_column_name`: Optional[Any], Column name to use for vector's in the table. Defaults to `'vector'`.   
+- `doc_id_key`: Optional[str], Column name to use for document id's in the table. Defaults to `'doc_id'`.  
+- `text_key`: Optional[str], Column name to use for text in the table. Defaults to `'text'`.  
+- `api_key`: Optional[str], API key to use for LanceDB cloud database. Defaults to `None`.  
+- `region`: Optional[str], Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
+- `nprobes` : Optional[int], Set the number of probes to use. Only applicable if ANN index is created on the table else its ignored. Defaults to `20`.
+- `refine_factor` : Optional[int], Refine the results by reading extra elements and re-ranking them in memory. Defaults to `None`.
+- `reranker`: Optional[Any], The reranker to use for LanceDB.
+        Defaults to `None`.
+- `overfetch_factor`: Optional[int], The factor by which to fetch more results.
+        Defaults to `1`.
+- `mode`: Optional[str], The mode to use for LanceDB.
+            Defaults to `"overwrite"`.
+- `query_type`:Optional[str], The type of query to use for LanceDB.
+            Defaults to `"vector"`.
+
+
+### Methods 
+
+- __from_table(cls, table: lancedb.db.LanceTable) -> `LanceDBVectorStore`__ : (class method) Creates instance from lancedb table. 
+
+- **_add_reranker(self, reranker: lancedb.rerankers.Reranker) -> `None`** : Add a reranker to an existing vector store. 
+    - Usage :
+        ```python
+        from lancedb.rerankers import ColbertReranker
+        reranker = ColbertReranker()
+        vector_store._add_reranker(reranker)
+        ```               
+- **_table_exists(self, tbl_name: `Optional[str]` = `None`) -> `bool`** : Returns `True` if `tbl_name` exists in database.
+- __create_index(  
+  self, scalar: `Optional[bool]` = False, col_name: `Optional[str]` = None, num_partitions: `Optional[int]` = 256, num_sub_vectors: `Optional[int]` = 96, index_cache_size: `Optional[int]` = None, metric: `Optional[str]` = "L2",  
+) -> `None`__ : Creates a scalar(for non-vector cols) or a vector index on a table.
+        Make sure your vector column has enough data before creating an index on it.
+
+- __add(self, nodes: `List[BaseNode]`, **add_kwargs: `Any`, ) -> `List[str]`__ :
+adds Nodes to the table
+
+- **delete(self, ref_doc_id: `str`) -> `None`**: Delete nodes using with node_ids.
+- **delete_nodes(self, node_ids: `List[str]`) -> `None`** : Delete nodes using with node_ids.
+- __query(
+        self,
+        query: `VectorStoreQuery`,
+        **kwargs: `Any`,
+    ) -> `VectorStoreQueryResult`__:
+        Query index(`VectorStoreIndex`) for top k most similar nodes. Accepts llamaIndex `VectorStoreQuery` object.

docs/src/migration.md

@@ -9,7 +9,8 @@ around the asynchronous client.
 This guide describes the differences between the two APIs and will hopefully assist users
 that would like to migrate to the new API.
 
-## Closeable Connections
+## Python
+### Closeable Connections
 
 The Connection now has a `close` method. You can call this when
 you are done with the connection to eagerly free resources. Currently
@@ -32,20 +33,20 @@ async def my_async_fn():
 It is not mandatory to call the `close` method. If you do not call it
 then the connection will be closed when the object is garbage collected.
 
-## Closeable Table
+### Closeable Table
 
 The Table now also has a `close` method, similar to the connection. This
 can be used to eagerly free the cache used by a Table object. Similar to
 the connection, it can be used as a context manager and it is not mandatory
 to call the `close` method.
 
-### Changes to Table APIs
+#### Changes to Table APIs
 
 - Previously `Table.schema` was a property. Now it is an async method.
 - The method `Table.__len__` was removed and `len(table)` will no longer
   work. Use `Table.count_rows` instead.
 
-### Creating Indices
+#### Creating Indices
 
 The `Table.create_index` method is now used for creating both vector indices
 and scalar indices. It currently requires a column name to be specified (the
@@ -55,12 +56,12 @@ the size of the data.
 To specify index configuration details you will need to specify which kind of
 index you are using.
 
-### Querying
+#### Querying
 
 The `Table.search` method has been renamed to `AsyncTable.vector_search` for
 clarity.
 
-## Features not yet supported
+### Features not yet supported
 
 The following features are not yet supported by the asynchronous API. However,
 we plan to support them soon.
@@ -74,3 +75,22 @@ we plan to support them soon.
   search
 - Remote connections to LanceDb Cloud are not yet supported.
 - The method Table.head is not yet supported.
+
+## TypeScript/JavaScript
+
+For JS/TS users, we offer a brand new SDK [@lancedb/lancedb](https://www.npmjs.com/package/@lancedb/lancedb)
+
+### Changes to Table APIs
+
+Previously `Table.schema` was a property. Now it is an async method.
+
+
+#### Creating Indices
+
+The `Table.createIndex` method is now used for creating both vector indices
+and scalar indices. It currently requires a column name to be specified (the
+column to index). Vector index defaults are now smarter and scale better with
+the size of the data.
+
+To specify index configuration details you will need to specify which kind of
+index you are using.

docs/src/notebooks/LlamaIndex_example.ipynb

@@ -0,0 +1,538 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "2db56c9b",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_index/blob/main/docs/docs/examples/vector_stores/LanceDBIndexDemo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "db0855d0",
+   "metadata": {},
+   "source": [
+    "# LanceDB Vector Store\n",
+    "In this notebook we are going to show how to use [LanceDB](https://www.lancedb.com) to perform vector searches in LlamaIndex"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "f44170b2",
+   "metadata": {},
+   "source": [
+    "If you're opening this Notebook on colab, you will probably need to install LlamaIndex 🦙."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6c84199c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install llama-index llama-index-vector-stores-lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a90ce34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install lancedb==0.6.13 #Only required if the above cell installs an older version of lancedb (pypi package may not be released yet)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39c62671",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Refresh vector store URI if restarting or re-using the same notebook\n",
+    "! rm -rf ./lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "59b54276",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import logging\n",
+    "import sys\n",
+    "\n",
+    "# Uncomment to see debug logs\n",
+    "# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n",
+    "# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\n",
+    "\n",
+    "\n",
+    "from llama_index.core import SimpleDirectoryReader, Document, StorageContext\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.vector_stores.lancedb import LanceDBVectorStore\n",
+    "import textwrap"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "26c71b6d",
+   "metadata": {},
+   "source": [
+    "### Setup OpenAI\n",
+    "The first step is to configure the openai key. It will be used to created embeddings for the documents loaded into the index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67b86621",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import openai\n",
+    "\n",
+    "openai.api_key = \"sk-\""
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "073f0a68",
+   "metadata": {},
+   "source": [
+    "Download Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eef1b911",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2024-06-11 16:42:37--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt\n",
+      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.108.133, ...\n",
+      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
+      "HTTP request sent, awaiting response... 200 OK\n",
+      "Length: 75042 (73K) [text/plain]\n",
+      "Saving to: ‘data/paul_graham/paul_graham_essay.txt’\n",
+      "\n",
+      "data/paul_graham/pa 100%[===================>]  73.28K  --.-KB/s    in 0.02s   \n",
+      "\n",
+      "2024-06-11 16:42:37 (3.97 MB/s) - ‘data/paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "!mkdir -p 'data/paul_graham/'\n",
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "f7010b1d-d1bb-4f08-9309-a328bb4ea396",
+   "metadata": {},
+   "source": [
+    "### Loading documents\n",
+    "Load the documents stored in the `data/paul_graham/` using the SimpleDirectoryReader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c154dd4b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Document ID: cac1ba78-5007-4cf8-89ba-280264790115 Document Hash: fe2d4d3ef3a860780f6c2599808caa587c8be6516fe0ba4ca53cf117044ba953\n"
+     ]
+    }
+   ],
+   "source": [
+    "documents = SimpleDirectoryReader(\"./data/paul_graham/\").load_data()\n",
+    "print(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "c0232fd1",
+   "metadata": {},
+   "source": [
+    "### Create the index\n",
+    "Here we create an index backed by LanceDB using the documents loaded previously. LanceDBVectorStore takes a few arguments.\n",
+    "- uri (str, required): Location where LanceDB will store its files.\n",
+    "- table_name (str, optional): The table name where the embeddings will be stored. Defaults to \"vectors\".\n",
+    "- nprobes (int, optional): The number of probes used. A higher number makes search more accurate but also slower. Defaults to 20.\n",
+    "- refine_factor: (int, optional): Refine the results by reading extra elements and re-ranking them in memory. Defaults to None\n",
+    "\n",
+    "- More details can be found at [LanceDB docs](https://lancedb.github.io/lancedb/ann_indexes)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f2e20ef",
+   "metadata": {},
+   "source": [
+    "##### For LanceDB cloud :\n",
+    "```python\n",
+    "vector_store = LanceDBVectorStore( \n",
+    "    uri=\"db://db_name\", # your remote DB URI\n",
+    "    api_key=\"sk_..\", # lancedb cloud api key\n",
+    "    region=\"your-region\" # the region you configured\n",
+    "    ...\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8731da62",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_store = LanceDBVectorStore(\n",
+    "    uri=\"./lancedb\", mode=\"overwrite\", query_type=\"hybrid\"\n",
+    ")\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "index = VectorStoreIndex.from_documents(\n",
+    "    documents, storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "8ee4473a-094f-4d0a-a825-e1213db07240",
+   "metadata": {},
+   "source": [
+    "### Query the index\n",
+    "We can now ask questions using our index. We can use filtering via `MetadataFilters` or use native lance `where` clause."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5eb6419b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.vector_stores import (\n",
+    "    MetadataFilters,\n",
+    "    FilterOperator,\n",
+    "    FilterCondition,\n",
+    "    MetadataFilter,\n",
+    ")\n",
+    "\n",
+    "from datetime import datetime\n",
+    "\n",
+    "\n",
+    "query_filters = MetadataFilters(\n",
+    "    filters=[\n",
+    "        MetadataFilter(\n",
+    "            key=\"creation_date\",\n",
+    "            operator=FilterOperator.EQ,\n",
+    "            value=datetime.now().strftime(\"%Y-%m-%d\"),\n",
+    "        ),\n",
+    "        MetadataFilter(\n",
+    "            key=\"file_size\", value=75040, operator=FilterOperator.GT\n",
+    "        ),\n",
+    "    ],\n",
+    "    condition=FilterCondition.AND,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ee201930",
+   "metadata": {},
+   "source": [
+    "### Hybrid Search\n",
+    "\n",
+    "LanceDB offers hybrid search with reranking capabilities. For complete documentation, refer [here](https://lancedb.github.io/lancedb/hybrid_search/hybrid_search/).\n",
+    "\n",
+    "This example uses the `colbert` reranker. The following cell installs the necessary dependencies for `colbert`. If you choose a different reranker, make sure to adjust the dependencies accordingly."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e12d1454",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U torch transformers tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c742cb07",
+   "metadata": {},
+   "source": [
+    "if you want to add a reranker at vector store initialization, you can pass it in the arguments like below :\n",
+    "```\n",
+    "from lancedb.rerankers import ColbertReranker\n",
+    "reranker = ColbertReranker()\n",
+    "vector_store = LanceDBVectorStore(uri=\"./lancedb\", reranker=reranker, mode=\"overwrite\")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27ea047b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8414517f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from lancedb.rerankers import ColbertReranker\n",
+    "\n",
+    "reranker = ColbertReranker()\n",
+    "vector_store._add_reranker(reranker)\n",
+    "\n",
+    "query_engine = index.as_query_engine(\n",
+    "    filters=query_filters,\n",
+    "    # vector_store_kwargs={\n",
+    "    #     \"query_type\": \"fts\",\n",
+    "    # },\n",
+    ")\n",
+    "\n",
+    "response = query_engine.query(\"How much did Viaweb charge per month?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc6ccb7a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Viaweb charged $100 a month for a small store and $300 a month for a big one.\n",
+      "metadata - {'65ed5f07-5b8a-4143-a939-e8764884828e': {'file_path': '/Users/raghavdixit/Desktop/open_source/llama_index_lance/docs/docs/examples/vector_stores/data/paul_graham/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-11', 'last_modified_date': '2024-06-11'}, 'be231827-20b8-4988-ac75-94fa79b3c22e': {'file_path': '/Users/raghavdixit/Desktop/open_source/llama_index_lance/docs/docs/examples/vector_stores/data/paul_graham/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-11', 'last_modified_date': '2024-06-11'}}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(response)\n",
+    "print(\"metadata -\", response.metadata)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c1c6c73",
+   "metadata": {},
+   "source": [
+    "##### lance filters(SQL like) directly via the `where` clause :"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a2bcc07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\n",
+    "retriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\n",
+    "response = retriever.retrieve(\"What did the author do growing up?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ac47cf9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "What I Worked On\n",
+      "\n",
+      "February 2021\n",
+      "\n",
+      "Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\n",
+      "\n",
+      "The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district's 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain's lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights.\n",
+      "\n",
+      "The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer.\n",
+      "\n",
+      "I was puzzled by the 1401. I couldn't figure out what to do with it. And in retrospect there's not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn't have any data stored on punched cards. The only other option was to do things that didn't rely on any input, like calculate approximations of pi, but I didn't know enough math to do anything interesting of that type. So I'm not surprised I can't remember any programs I wrote, because they can't have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn't. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager's expression made clear.\n",
+      "\n",
+      "With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1]\n",
+      "\n",
+      "The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer.\n",
+      "\n",
+      "Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he'd write 2 pages at a time and then print them out, but it was a lot better than a typewriter.\n",
+      "\n",
+      "Though I liked programming, I didn't plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn't much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored.\n",
+      "\n",
+      "I couldn't have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI.\n",
+      "\n",
+      "AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven't tried rereading The Moon is a Harsh Mistress, so I don't know how well it has aged, but when I read it I was drawn entirely into its world.\n",
+      "metadata - {'file_path': '/Users/raghavdixit/Desktop/open_source/llama_index_lance/docs/docs/examples/vector_stores/data/paul_graham/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-11', 'last_modified_date': '2024-06-11'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(response[0].get_content())\n",
+    "print(\"metadata -\", response[0].metadata)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "6afc84ac",
+   "metadata": {},
+   "source": [
+    "### Appending data\n",
+    "You can also add data to an existing index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "759a532e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nodes = [node.node for node in response]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "069fc099",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del index\n",
+    "\n",
+    "index = VectorStoreIndex.from_documents(\n",
+    "    [Document(text=\"The sky is purple in Portland, Maine\")],\n",
+    "    uri=\"/tmp/new_dataset\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a64ed441",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.insert_nodes(nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b5cffcfe",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Portland, Maine\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "response = query_engine.query(\"Where is the sky purple?\")\n",
+    "print(textwrap.fill(str(response), 100))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ec548a02",
+   "metadata": {},
+   "source": [
+    "You can also create an index from an existing table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc99404d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del index\n",
+    "\n",
+    "vec_store = LanceDBVectorStore.from_table(vector_store._table)\n",
+    "index = VectorStoreIndex.from_vector_store(vec_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7b2e8cca",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The author started Viaweb and Aspra.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "response = query_engine.query(\"What companies did the author start?\")\n",
+    "print(textwrap.fill(str(response), 100))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/src/notebooks/langchain_example.ipynb

@@ -0,0 +1,566 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "683953b3",
+   "metadata": {},
+   "source": [
+    "# LanceDB\n",
+    "\n",
+    ">[LanceDB](https://lancedb.com/) is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings. Fully open source.\n",
+    "\n",
+    "This notebook shows how to use functionality related to the `LanceDB` vector database based on the Lance data format."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b1051ba9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install tantivy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "88ac92c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U langchain-openai langchain-community"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a1c84d6-a10f-428c-95cd-46d3a1702e07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install lancedb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "99134dd1-b91e-486f-8d90-534248e43b9d",
+   "metadata": {},
+   "source": [
+    "We want to use OpenAIEmbeddings so we have to get the OpenAI API Key. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "a0361f5c-e6f4-45f4-b829-11680cf03cec",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d114ed78",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! rm -rf /tmp/lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a3c3999a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import TextLoader\n",
+    "from langchain_community.vectorstores import LanceDB\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
+    "\n",
+    "loader = TextLoader(\"../../how_to/state_of_the_union.txt\")\n",
+    "documents = loader.load()\n",
+    "\n",
+    "documents = CharacterTextSplitter().split_documents(documents)\n",
+    "embeddings = OpenAIEmbeddings()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e9517bb0",
+   "metadata": {},
+   "source": [
+    "##### For LanceDB cloud, you can invoke the vector store as follows :\n",
+    "\n",
+    "\n",
+    "```python\n",
+    "db_url = \"db://lang_test\" # url of db you created\n",
+    "api_key = \"xxxxx\" # your API key\n",
+    "region=\"us-east-1-dev\"  # your selected region\n",
+    "\n",
+    "vector_store = LanceDB(\n",
+    "    uri=db_url,\n",
+    "    api_key=api_key,\n",
+    "    region=region,\n",
+    "    embedding=embeddings,\n",
+    "    table_name='langchain_test'\n",
+    "    )\n",
+    "```\n",
+    "\n",
+    "You can also add `region`, `api_key`, `uri` to `from_documents()` classmethod\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6e104aee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from lancedb.rerankers import LinearCombinationReranker\n",
+    "\n",
+    "reranker = LinearCombinationReranker(weight=0.3)\n",
+    "\n",
+    "docsearch = LanceDB.from_documents(documents, embeddings, reranker=reranker)\n",
+    "query = \"What did the president say about Ketanji Brown Jackson\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "259c7988",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "relevance score -  0.7066475030191711\n",
+      "text-  They were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \n",
+      "\n",
+      "Officer Mora was 27 years old. \n",
+      "\n",
+      "Officer Rivera was 22. \n",
+      "\n",
+      "Both Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \n",
+      "\n",
+      "I spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves. \n",
+      "\n",
+      "I’ve worked on these issues a long time. \n",
+      "\n",
+      "I know what works: Investing in crime prevention and community police officers who’ll walk the beat, who’ll know the neighborhood, and who can restore trust and safety. \n",
+      "\n",
+      "So let’s not abandon our streets. Or choose between safety and equal justice. \n",
+      "\n",
+      "Let’s come together to protect our communities, restore trust, and hold law enforcement accountable. \n",
+      "\n",
+      "That’s why the Justice Department required body cameras, banned chokeholds, and restricted no-knock warrants for its officers. \n",
+      "\n",
+      "That’s why the American Rescue \n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = docsearch.similarity_search_with_relevance_scores(query)\n",
+    "print(\"relevance score - \", docs[0][1])\n",
+    "print(\"text- \", docs[0][0].page_content[:1000])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "9fa29dae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "distance -  0.30000001192092896\n",
+      "text-  My administration is providing assistance with job training and housing, and now helping lower-income veterans get VA care debt-free.  \n",
+      "\n",
+      "Our troops in Iraq and Afghanistan faced many dangers. \n",
+      "\n",
+      "One was stationed at bases and breathing in toxic smoke from “burn pits” that incinerated wastes of war—medical and hazard material, jet fuel, and more. \n",
+      "\n",
+      "When they came home, many of the world’s fittest and best trained warriors were never the same. \n",
+      "\n",
+      "Headaches. Numbness. Dizziness. \n",
+      "\n",
+      "A cancer that would put them in a flag-draped coffin. \n",
+      "\n",
+      "I know. \n",
+      "\n",
+      "One of those soldiers was my son Major Beau Biden. \n",
+      "\n",
+      "We don’t know for sure if a burn pit was the cause of his brain cancer, or the diseases of so many of our troops. \n",
+      "\n",
+      "But I’m committed to finding out everything we can. \n",
+      "\n",
+      "Committed to military families like Danielle Robinson from Ohio. \n",
+      "\n",
+      "The widow of Sergeant First Class Heath Robinson.  \n",
+      "\n",
+      "He was born a soldier. Army National Guard. Combat medic in Kosovo and Iraq. \n",
+      "\n",
+      "Stationed near Baghdad, just ya\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = docsearch.similarity_search_with_score(query=\"Headaches\", query_type=\"hybrid\")\n",
+    "print(\"distance - \", docs[0][1])\n",
+    "print(\"text- \", docs[0][0].page_content[:1000])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "e70ad201",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "reranker :  <lancedb.rerankers.linear_combination.LinearCombinationReranker object at 0x107ef1130>\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"reranker : \", docsearch._reranker)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f5e1cdfd",
+   "metadata": {},
+   "source": [
+    "Additionaly, to explore the table you can load it into a df or save it in a csv file: \n",
+    "```python\n",
+    "tbl = docsearch.get_table()\n",
+    "print(\"tbl:\", tbl)\n",
+    "pd_df = tbl.to_pandas()\n",
+    "# pd_df.to_csv(\"docsearch.csv\", index=False)\n",
+    "\n",
+    "# you can also create a new vector store object using an older connection object:\n",
+    "vector_store = LanceDB(connection=tbl, embedding=embeddings)\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "9c608226",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "metadata : {'source': '../../how_to/state_of_the_union.txt'}\n",
+      "\n",
+      "SQL filtering :\n",
+      "\n",
+      "They were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \n",
+      "\n",
+      "Officer Mora was 27 years old. \n",
+      "\n",
+      "Officer Rivera was 22. \n",
+      "\n",
+      "Both Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \n",
+      "\n",
+      "I spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves. \n",
+      "\n",
+      "I’ve worked on these issues a long time. \n",
+      "\n",
+      "I know what works: Investing in crime prevention and community police officers who’ll walk the beat, who’ll know the neighborhood, and who can restore trust and safety. \n",
+      "\n",
+      "So let’s not abandon our streets. Or choose between safety and equal justice. \n",
+      "\n",
+      "Let’s come together to protect our communities, restore trust, and hold law enforcement accountable. \n",
+      "\n",
+      "That’s why the Justice Department required body cameras, banned chokeholds, and restricted no-knock warrants for its officers. \n",
+      "\n",
+      "That’s why the American Rescue Plan provided $350 Billion that cities, states, and counties can use to hire more police and invest in proven strategies like community violence interruption—trusted messengers breaking the cycle of violence and trauma and giving young people hope.  \n",
+      "\n",
+      "We should all agree: The answer is not to Defund the police. The answer is to FUND the police with the resources and training they need to protect our communities. \n",
+      "\n",
+      "I ask Democrats and Republicans alike: Pass my budget and keep our neighborhoods safe.  \n",
+      "\n",
+      "And I will keep doing everything in my power to crack down on gun trafficking and ghost guns you can buy online and make at home—they have no serial numbers and can’t be traced. \n",
+      "\n",
+      "And I ask Congress to pass proven measures to reduce gun violence. Pass universal background checks. Why should anyone on a terrorist list be able to purchase a weapon? \n",
+      "\n",
+      "Ban assault weapons and high-capacity magazines. \n",
+      "\n",
+      "Repeal the liability shield that makes gun manufacturers the only industry in America that can’t be sued. \n",
+      "\n",
+      "These laws don’t infringe on the Second Amendment. They save lives. \n",
+      "\n",
+      "The most fundamental right in America is the right to vote – and to have it counted. And it’s under assault. \n",
+      "\n",
+      "In state after state, new laws have been passed, not only to suppress the vote, but to subvert entire elections. \n",
+      "\n",
+      "We cannot let this happen. \n",
+      "\n",
+      "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n",
+      "\n",
+      "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n",
+      "\n",
+      "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n",
+      "\n",
+      "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. \n",
+      "\n",
+      "A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \n",
+      "\n",
+      "And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \n",
+      "\n",
+      "We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling.  \n",
+      "\n",
+      "We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers.  \n",
+      "\n",
+      "We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster.\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = docsearch.similarity_search(\n",
+    "    query=query, filter={\"metadata.source\": \"../../how_to/state_of_the_union.txt\"}\n",
+    ")\n",
+    "\n",
+    "print(\"metadata :\", docs[0].metadata)\n",
+    "\n",
+    "# or you can directly supply SQL string filters :\n",
+    "\n",
+    "print(\"\\nSQL filtering :\\n\")\n",
+    "docs = docsearch.similarity_search(query=query, filter=\"text LIKE '%Officer Rivera%'\")\n",
+    "print(docs[0].page_content)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9a173c94",
+   "metadata": {},
+   "source": [
+    "## Adding images "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05f669d7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U langchain-experimental"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ed69810",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install open_clip_torch torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "2cacb5ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! rm -rf '/tmp/multimmodal_lance'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "b3456e2c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_experimental.open_clip import OpenCLIPEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "3848eba2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "import requests\n",
+    "\n",
+    "# List of image URLs to download\n",
+    "image_urls = [\n",
+    "    \"https://github.com/raghavdixit99/assets/assets/34462078/abf47cc4-d979-4aaa-83be-53a2115bf318\",\n",
+    "    \"https://github.com/raghavdixit99/assets/assets/34462078/93be928e-522b-4e37-889d-d4efd54b2112\",\n",
+    "]\n",
+    "\n",
+    "texts = [\"bird\", \"dragon\"]\n",
+    "\n",
+    "# Directory to save images\n",
+    "dir_name = \"./photos/\"\n",
+    "\n",
+    "# Create directory if it doesn't exist\n",
+    "os.makedirs(dir_name, exist_ok=True)\n",
+    "\n",
+    "image_uris = []\n",
+    "# Download and save each image\n",
+    "for i, url in enumerate(image_urls, start=1):\n",
+    "    response = requests.get(url)\n",
+    "    path = os.path.join(dir_name, f\"image{i}.jpg\")\n",
+    "    image_uris.append(path)\n",
+    "    with open(path, \"wb\") as f:\n",
+    "        f.write(response.content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "3d62c2a0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.vectorstores import LanceDB\n",
+    "\n",
+    "vec_store = LanceDB(\n",
+    "    table_name=\"multimodal_test\",\n",
+    "    embedding=OpenCLIPEmbeddings(),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "ebbb4881",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['b673620b-01f0-42ca-a92e-d033bb92c0a6',\n",
+       " '99c3a5b0-b577-417a-8177-92f4a655dbfb']"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store.add_images(uris=image_uris)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "3c29dea3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['f7adde5d-a4a3-402b-9e73-088b230722c3',\n",
+       " 'cbed59da-0aec-4bff-8820-9e59d81a2140']"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store.add_texts(texts)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "8b2f25ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "img_embed = vec_store._embedding.embed_query(\"bird\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "87a24079",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Document(page_content='bird', metadata={'id': 'f7adde5d-a4a3-402b-9e73-088b230722c3'})"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store.similarity_search_by_vector(img_embed)[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "78557867",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LanceTable(connection=LanceDBConnection(/tmp/lancedb), name=\"multimodal_test\")"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store._table"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/src/reranking/index.md

@@ -15,7 +15,6 @@ LanceDB comes with some built-in rerankers. Some of the rerankers that are avail
 Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder.
 
 ```python
-import numpy
 import lancedb
 from lancedb.embeddings import get_registry
 from lancedb.pydantic import LanceModel, Vector
@@ -54,6 +53,7 @@ LanceDB comes with some built-in rerankers. Here are some of the rerankers that
 - [ColBERT Reranker](./colbert.md)
 - [OpenAI Reranker](./openai.md)
 - [Linear Combination Reranker](./linear_combination.md)
+- [Jina Reranker](./jina.md)
 
 ## Creating Custom Rerankers
 

docs/src/reranking/jina.md

@@ -0,0 +1,78 @@
+# Jina Reranker
+
+This re-ranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this re-ranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+
+
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+
+```python
+import os
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import JinaReranker
+
+os.environ['JINA_API_KEY'] = "jina_*"
+
+
+embedder = get_registry().get("jina").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = JinaReranker(api_key="key")
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker/|
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
+| `api_key` | `str` | `None` | The API key for the Jina API. If not provided, the `JINA_API_KEY` environment variable is used. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/search.md

@@ -53,13 +53,24 @@ db.create_table("my_vectors", data=data)
         .to_list()
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/search_legacy.ts:import"
 
-    --8<-- "docs/src/search_legacy.ts:search1"
+        ```ts
-    ```
+        --8<-- "nodejs/examples/search.ts:import"
+
+        --8<-- "nodejs/examples/search.ts:search1"
+        ```
+
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/search_legacy.ts:import"
+
+        --8<-- "docs/src/search_legacy.ts:search1"
+        ```
 
 By default, `l2` will be used as metric type. You can specify the metric type as
 `cosine` or `dot` if required.
@@ -73,11 +84,19 @@ By default, `l2` will be used as metric type. You can specify the metric type as
         .to_list()
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/search_legacy.ts:search2"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/search.ts:search2"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```javascript
+        --8<-- "docs/src/search_legacy.ts:search2"
+        ```
 
 ## Approximate nearest neighbor (ANN) search
 

docs/src/sql.md

@@ -44,11 +44,19 @@ const tbl = await db.createTable('myVectors', data)
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/sql_legacy.ts:search"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/filtering.ts:search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/sql_legacy.ts:search"
+        ```
 
 ## SQL filters
 
@@ -78,11 +86,19 @@ For example, the following filter string is acceptable:
        .to_arrow()
     ```
 
-=== "Javascript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/sql_legacy.ts:vec_search"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/filtering.ts:vec_search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/sql_legacy.ts:vec_search"
+        ```
 
 If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
 you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
@@ -148,10 +164,18 @@ You can also filter your data without search.
     tbl.search().where("id = 10").limit(10).to_arrow()
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<---- "docs/src/sql_legacy.ts:sql_search"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/filtering.ts:sql_search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<---- "docs/src/sql_legacy.ts:sql_search"
+        ```
 
 !!!warning "If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set."

docs/test/md_testing.py

@@ -7,8 +7,7 @@ excluded_globs = [
     "../src/fts.md",
     "../src/embedding.md",
     "../src/examples/*.md",
-    "../src/integrations/voxel51.md",
+    "../src/integrations/*.md",
-    "../src/integrations/langchain.md",
     "../src/guides/tables.md",
     "../src/python/duckdb.md",
     "../src/embeddings/*.md",
@@ -17,6 +16,7 @@ excluded_globs = [
     "../src/basic.md",
     "../src/hybrid_search/hybrid_search.md",
     "../src/reranking/*.md",
+    "../src/guides/tuning_retrievers/*.md",
 ]
 
 python_prefix = "py"

java/core/lancedb-jni/src/ffi.rs

@@ -175,8 +175,8 @@ impl JNIEnvExt for JNIEnv<'_> {
         if obj.is_null() {
             return Ok(None);
         }
-        let is_empty = self.call_method(obj, "isEmpty", "()Z", &[])?;
+        let is_present = self.call_method(obj, "isPresent", "()Z", &[])?;
-        if is_empty.z()? {
+        if !is_present.z()? {
             // TODO(lu): put get java object into here cuz can only get java Object
             Ok(None)
         } else {

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.5.1",
+  "version": "0.7.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.5.1",
+      "version": "0.7.0",
       "cpu": [
         "x64",
         "arm64"

node/package.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.5.1",
+  "version": "0.7.0",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb-node index.node -- cargo build --message-format=json",
+    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb_node index.node -- cargo build -p lancedb-node --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
     "integration-test": "npm run tsc && mocha -recursive dist/integration_test",

node/src/index.ts

@@ -62,6 +62,8 @@ export {
 
 const defaultAwsRegion = "us-west-2";
 
+const defaultRequestTimeout = 10_000
+
 export interface AwsCredentials {
   accessKeyId: string
 
@@ -119,6 +121,11 @@ export interface ConnectionOptions {
    */
   hostOverride?: string
 
+  /**
+   * Duration in milliseconds for request timeout. Default = 10,000 (10 seconds)
+   */
+  timeout?: number
+
   /**
    * (For LanceDB OSS only): The interval, in seconds, at which to check for
    * updates to the table from other processes. If None, then consistency is not
@@ -204,7 +211,8 @@ export async function connect(
           awsCredentials: undefined,
           awsRegion: defaultAwsRegion,
           apiKey: undefined,
-          region: defaultAwsRegion
+          region: defaultAwsRegion,
+          timeout: defaultRequestTimeout 
         },
         arg
       );
@@ -695,18 +703,26 @@ export interface MergeInsertArgs {
   whenNotMatchedBySourceDelete?: string | boolean
 }
 
+export enum IndexStatus {
+  Pending = "pending",
+  Indexing = "indexing",
+  Done = "done",
+  Failed = "failed"
+}
+
 export interface VectorIndex {
   columns: string[]
   name: string
   uuid: string
+  status: IndexStatus
 }
 
 export interface IndexStats {
   numIndexedRows: number | null
   numUnindexedRows: number | null
-  index_type: string | null
+  indexType: string | null
-  distance_type: string | null
+  distanceType: string | null
-  completed_at: string | null
+  completedAt: string | null
 }
 
 /**

node/src/remote/client.ts

@@ -41,7 +41,7 @@ async function callWithMiddlewares (
     if (i > middlewares.length) {
       const headers = Object.fromEntries(req.headers.entries())
       const params = Object.fromEntries(req.params?.entries() ?? [])
-      const timeout = 10000
+      const timeout = opts?.timeout
       let res
       if (req.method === Method.POST) {
         res = await axios.post(
@@ -82,6 +82,7 @@ async function callWithMiddlewares (
 
 interface MiddlewareInvocationOptions {
   responseType?: ResponseType
+  timeout?: number,
 }
 
 /**
@@ -123,15 +124,19 @@ export class HttpLancedbClient {
   private readonly _url: string
   private readonly _apiKey: () => string
   private readonly _middlewares: HttpLancedbClientMiddleware[]
+  private readonly _timeout: number | undefined
 
   public constructor (
     url: string,
     apiKey: string,
-    private readonly _dbName?: string
+    timeout?: number,
+    private readonly _dbName?: string,
+    
   ) {
     this._url = url
     this._apiKey = () => apiKey
     this._middlewares = []
+    this._timeout = timeout
   }
 
   get uri (): string {
@@ -230,7 +235,10 @@ export class HttpLancedbClient {
 
     let response
     try {
-      response = await callWithMiddlewares(req, this._middlewares, { responseType })
+      response = await callWithMiddlewares(req, this._middlewares, {
+        responseType,
+        timeout: this._timeout,
+      })
 
       // return response
     } catch (err: any) {
@@ -267,7 +275,7 @@ export class HttpLancedbClient {
    * Make a clone of this client
    */
   private clone (): HttpLancedbClient {
-    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._dbName)
+    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._timeout, this._dbName)
     for (const mw of this._middlewares) {
       clone._middlewares.push(mw)
     }

node/src/remote/index.ts

@@ -72,6 +72,7 @@ export class RemoteConnection implements Connection {
     this._client = new HttpLancedbClient(
       server,
       opts.apiKey,
+      opts.timeout,
       opts.hostOverride === undefined ? undefined : this._dbName
     )
   }
@@ -522,9 +523,9 @@ export class RemoteTable<T = number[]> implements Table<T> {
     return {
       numIndexedRows: body?.num_indexed_rows,
       numUnindexedRows: body?.num_unindexed_rows,
-      index_type: body?.index_type,
+      indexType: body?.index_type,
-      distance_type: body?.distance_type,
+      distanceType: body?.distance_type,
-      completed_at: body?.completed_at
+      completedAt: body?.completed_at
     }
   }
 

nodejs/Cargo.toml

@@ -15,11 +15,11 @@ crate-type = ["cdylib"]
 arrow-ipc.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb" }
-napi = { version = "2.15", default-features = false, features = [
+napi = { version = "2.16.8", default-features = false, features = [
-    "napi7",
+    "napi9",
     "async",
 ] }
-napi-derive = "2"
+napi-derive = "2.16.4"
 
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "*", features = ["static"] }

nodejs/__test__/connection.test.ts

@@ -57,6 +57,18 @@ describe("given a connection", () => {
     expect(db.isOpen()).toBe(false);
     await expect(db.tableNames()).rejects.toThrow("Connection is closed");
   });
+  it("should be able to create a table from an object arg `createTable(options)`, or args `createTable(name, data, options)`", async () => {
+    let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
+    await expect(tbl.countRows()).resolves.toBe(2);
+
+    tbl = await db.createTable({
+      name: "test",
+      data: [{ id: 3 }],
+      mode: "overwrite",
+    });
+
+    await expect(tbl.countRows()).resolves.toBe(1);
+  });
 
   it("should fail if creating table twice, unless overwrite is true", async () => {
     let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);

nodejs/__test__/embedding.test.ts

@@ -230,7 +230,7 @@ describe("embedding functions", () => {
     },
   );
 
-  test.only.each([new Float16(), new Float32(), new Float64()])(
+  test.each([new Float16(), new Float32(), new Float64()])(
     "should be able to provide auto embeddings with multiple float datatypes",
     async (floatType) => {
       @register("test1")

nodejs/__test__/registry.test.ts

@@ -63,6 +63,7 @@ describe("Registry", () => {
         return data.map(() => [1, 2, 3]);
       }
     }
+
     const func = getRegistry()
       .get<MockEmbeddingFunction>("mock-embedding")!
       .create();

nodejs/__test__/s3_integration.test.ts

@@ -14,6 +14,11 @@
 
 /* eslint-disable @typescript-eslint/naming-convention */
 
+import {
+  CreateTableCommand,
+  DeleteTableCommand,
+  DynamoDBClient,
+} from "@aws-sdk/client-dynamodb";
 import {
   CreateKeyCommand,
   KMSClient,
@@ -38,6 +43,7 @@ const CONFIG = {
   awsAccessKeyId: "ACCESSKEY",
   awsSecretAccessKey: "SECRETKEY",
   awsEndpoint: "http://127.0.0.1:4566",
+  dynamodbEndpoint: "http://127.0.0.1:4566",
   awsRegion: "us-east-1",
 };
 
@@ -66,7 +72,6 @@ class S3Bucket {
     } catch {
       // It's fine if the bucket doesn't exist
     }
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new CreateBucketCommand({ Bucket: name }));
     return new S3Bucket(name);
   }
@@ -79,32 +84,27 @@ class S3Bucket {
   static async deleteBucket(client: S3Client, name: string) {
     // Must delete all objects before we can delete the bucket
     const objects = await client.send(
-      // biome-ignore lint/style/useNamingConvention: we dont control s3's api
       new ListObjectsV2Command({ Bucket: name }),
     );
     if (objects.Contents) {
       for (const object of objects.Contents) {
         await client.send(
-          // biome-ignore lint/style/useNamingConvention: we dont control s3's api
           new DeleteObjectCommand({ Bucket: name, Key: object.Key }),
         );
       }
     }
 
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new DeleteBucketCommand({ Bucket: name }));
   }
 
   public async assertAllEncrypted(path: string, keyId: string) {
     const client = S3Bucket.s3Client();
     const objects = await client.send(
-      // biome-ignore lint/style/useNamingConvention: we dont control s3's api
       new ListObjectsV2Command({ Bucket: this.name, Prefix: path }),
     );
     if (objects.Contents) {
       for (const object of objects.Contents) {
         const metadata = await client.send(
-          // biome-ignore lint/style/useNamingConvention: we dont control s3's api
           new HeadObjectCommand({ Bucket: this.name, Key: object.Key }),
         );
         expect(metadata.ServerSideEncryption).toBe("aws:kms");
@@ -143,7 +143,6 @@ class KmsKey {
 
   public async delete() {
     const client = KmsKey.kmsClient();
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new ScheduleKeyDeletionCommand({ KeyId: this.keyId }));
   }
 }
@@ -224,3 +223,91 @@ maybeDescribe("storage_options", () => {
     await bucket.assertAllEncrypted("test/table2.lance", kmsKey.keyId);
   });
 });
+
+class DynamoDBCommitTable {
+  name: string;
+  constructor(name: string) {
+    this.name = name;
+  }
+
+  static dynamoClient() {
+    return new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+  }
+
+  public static async create(name: string): Promise<DynamoDBCommitTable> {
+    const client = DynamoDBCommitTable.dynamoClient();
+    const command = new CreateTableCommand({
+      TableName: name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    return new DynamoDBCommitTable(name);
+  }
+
+  public async delete() {
+    const client = DynamoDBCommitTable.dynamoClient();
+    await client.send(new DeleteTableCommand({ TableName: this.name }));
+  }
+}
+
+maybeDescribe("DynamoDB Lock", () => {
+  let bucket: S3Bucket;
+  let commitTable: DynamoDBCommitTable;
+
+  beforeAll(async () => {
+    bucket = await S3Bucket.create("lancedb2");
+    commitTable = await DynamoDBCommitTable.create("commitTable");
+  });
+
+  afterAll(async () => {
+    await commitTable.delete();
+    await bucket.delete();
+  });
+
+  it("can be used to configure a DynamoDB table for commit log", async () => {
+    const uri = `s3+ddb://${bucket.name}/test?ddbTableName=${commitTable.name}`;
+    const db = await connect(uri, {
+      storageOptions: CONFIG,
+      readConsistencyInterval: 0,
+    });
+
+    const table = await db.createTable("test", [{ a: 1, b: 2 }]);
+
+    // 5 concurrent appends
+    const futs = Array.from({ length: 5 }, async () => {
+      // Open a table so each append has a separate table reference. Otherwise
+      // they will share the same table reference and the internal ReadWriteLock
+      // will prevent any real concurrency.
+      const table = await db.openTable("test");
+      await table.add([{ a: 2, b: 3 }]);
+    });
+    await Promise.all(futs);
+
+    const rowCount = await table.countRows();
+    expect(rowCount).toBe(6);
+  });
+});

nodejs/__test__/table.test.ts

@@ -39,7 +39,9 @@ describe.each([arrow, arrowOld])("Given a table", (arrow: any) => {
   let tmpDir: tmp.DirResult;
   let table: Table;
 
-  const schema = new arrow.Schema([
+  const schema:
+    | import("apache-arrow").Schema
+    | import("apache-arrow-old").Schema = new arrow.Schema([
     new arrow.Field("id", new arrow.Float64(), true),
   ]);
 
@@ -94,6 +96,50 @@ describe.each([arrow, arrowOld])("Given a table", (arrow: any) => {
     expect(await table.countRows("id == 10")).toBe(1);
   });
 
+  it("should let me update values with `values`", async () => {
+    await table.add([{ id: 1 }]);
+    expect(await table.countRows("id == 1")).toBe(1);
+    expect(await table.countRows("id == 7")).toBe(0);
+    await table.update({ values: { id: 7 } });
+    expect(await table.countRows("id == 1")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    await table.add([{ id: 2 }]);
+    // Test Map as input
+    await table.update({
+      values: {
+        id: "10",
+      },
+      where: "id % 2 == 0",
+    });
+    expect(await table.countRows("id == 2")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    expect(await table.countRows("id == 10")).toBe(1);
+  });
+
+  it("should let me update values with `valuesSql`", async () => {
+    await table.add([{ id: 1 }]);
+    expect(await table.countRows("id == 1")).toBe(1);
+    expect(await table.countRows("id == 7")).toBe(0);
+    await table.update({
+      valuesSql: {
+        id: "7",
+      },
+    });
+    expect(await table.countRows("id == 1")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    await table.add([{ id: 2 }]);
+    // Test Map as input
+    await table.update({
+      valuesSql: {
+        id: "10",
+      },
+      where: "id % 2 == 0",
+    });
+    expect(await table.countRows("id == 2")).toBe(0);
+    expect(await table.countRows("id == 7")).toBe(1);
+    expect(await table.countRows("id == 10")).toBe(1);
+  });
+
   // https://github.com/lancedb/lancedb/issues/1293
   test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
     "can create empty table with non default float type: %s",
@@ -132,6 +178,140 @@ describe.each([arrow, arrowOld])("Given a table", (arrow: any) => {
   });
 });
 
+describe("merge insert", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const conn = await connect(tmpDir.name);
+
+    table = await conn.createTable("some_table", [
+      { a: 1, b: "a" },
+      { a: 2, b: "b" },
+      { a: 3, b: "c" },
+    ]);
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  test("upsert", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute(newData);
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+
+    expect(
+      JSON.parse(JSON.stringify((await table.toArrow()).toArray())),
+    ).toEqual(expected);
+  });
+  test("conditional update", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll({ where: "target.b = 'b'" })
+      .execute(newData);
+
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "x" },
+      { a: 3, b: "c" },
+    ];
+    // round trip to arrow and back to json to avoid comparing arrow objects to js object
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+
+    expect(res).toEqual(expected);
+  });
+
+  test("insert if not exists", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 3, b: "y" },
+      { a: 4, b: "z" },
+    ];
+    await table.mergeInsert("a").whenNotMatchedInsertAll().execute(newData);
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "b" },
+      { a: 3, b: "c" },
+      { a: 4, b: "z" },
+    ];
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+    expect(res).toEqual(expected);
+  });
+  test("replace range", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .whenNotMatchedBySourceDelete({ where: "a > 2" })
+      .execute(newData);
+
+    const expected = [
+      { a: 1, b: "a" },
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+    expect(res).toEqual(expected);
+  });
+  test("replace range no condition", async () => {
+    const newData = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+    await table
+      .mergeInsert("a")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .whenNotMatchedBySourceDelete()
+      .execute(newData);
+
+    const expected = [
+      { a: 2, b: "x" },
+      { a: 4, b: "z" },
+    ];
+
+    // biome-ignore lint/suspicious/noExplicitAny: test
+    let res: any[] = JSON.parse(
+      JSON.stringify((await table.toArrow()).toArray()),
+    );
+    res = res.sort((a, b) => a.a - b.a);
+    expect(res).toEqual(expected);
+  });
+});
+
 describe("When creating an index", () => {
   let tmpDir: tmp.DirResult;
   const schema = new Schema([
@@ -171,6 +351,7 @@ describe("When creating an index", () => {
     const indices = await tbl.listIndices();
     expect(indices.length).toBe(1);
     expect(indices[0]).toEqual({
+      name: "vec_idx",
       indexType: "IvfPq",
       columns: ["vec"],
     });
@@ -180,7 +361,7 @@ describe("When creating an index", () => {
       .query()
       .limit(2)
       .nearestTo(queryVec)
-      .distanceType("DoT")
+      .distanceType("dot")
       .toArrow();
     expect(rst.numRows).toBe(2);
 
@@ -227,6 +408,24 @@ describe("When creating an index", () => {
     for await (const r of tbl.query().where("id > 1").select(["id"])) {
       expect(r.numRows).toBe(298);
     }
+    // should also work with 'filter' alias
+    for await (const r of tbl.query().filter("id > 1").select(["id"])) {
+      expect(r.numRows).toBe(298);
+    }
+  });
+
+  test("should be able to get index stats", async () => {
+    await tbl.createIndex("id");
+
+    const stats = await tbl.indexStats("id_idx");
+    expect(stats).toBeDefined();
+    expect(stats?.numIndexedRows).toEqual(300);
+    expect(stats?.numUnindexedRows).toEqual(0);
+  });
+
+  test("when getting stats on non-existent index", async () => {
+    const stats = await tbl.indexStats("some non-existent index");
+    expect(stats).toBeUndefined();
   });
 
   // TODO: Move this test to the query API test (making sure we can reject queries
@@ -551,10 +750,10 @@ describe("table.search", () => {
     const data = [{ text: "hello world" }, { text: "goodbye world" }];
     const table = await db.createTable("test", data, { schema });
 
-    const results = await table.search("greetings").then((r) => r.toArray());
+    const results = await table.search("greetings").toArray();
     expect(results[0].text).toBe(data[0].text);
 
-    const results2 = await table.search("farewell").then((r) => r.toArray());
+    const results2 = await table.search("farewell").toArray();
     expect(results2[0].text).toBe(data[1].text);
   });
 
@@ -566,7 +765,7 @@ describe("table.search", () => {
     ];
     const table = await db.createTable("test", data);
 
-    expect(table.search("hello")).rejects.toThrow(
+    expect(table.search("hello").toArray()).rejects.toThrow(
       "No embedding functions are defined in the table",
     );
   });
@@ -590,3 +789,27 @@ describe("table.search", () => {
     expect(results[0].text).toBe(data[1].text);
   });
 });
+
+describe("when calling explainPlan", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+  let queryVec: number[];
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const con = await connect(tmpDir.name);
+    table = await con.createTable("vectors", [{ id: 1, vector: [0.1, 0.2] }]);
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("retrieves query plan", async () => {
+    queryVec = Array(2)
+      .fill(1)
+      .map(() => Math.random());
+    const plan = await table.query().nearestTo(queryVec).explainPlan(true);
+
+    expect(plan).toMatch("KNN");
+  });
+});

nodejs/__test__/tsconfig.json

@@ -6,5 +6,5 @@
     "target": "es2022",
     "types": ["jest", "node"]
   },
-  "include": ["**/*"]
+  "include": ["**/*", "../examples/ann_indexes.ts"]
 }

nodejs/__test__/util.test.ts

@@ -0,0 +1,28 @@
+import { IntoSql, toSQL } from "../lancedb/util";
+test.each([
+  ["string", "'string'"],
+  [123, "123"],
+  [1.11, "1.11"],
+  [true, "TRUE"],
+  [false, "FALSE"],
+  [null, "NULL"],
+  [new Date("2021-01-01T00:00:00.000Z"), "'2021-01-01T00:00:00.000Z'"],
+  [[1, 2, 3], "[1, 2, 3]"],
+  [new ArrayBuffer(8), "X'0000000000000000'"],
+  [Buffer.from("hello"), "X'68656c6c6f'"],
+  ["Hello 'world'", "'Hello ''world'''"],
+])("toSQL(%p) === %p", (value, expected) => {
+  expect(toSQL(value)).toBe(expected);
+});
+
+test("toSQL({}) throws on unsupported value type", () => {
+  expect(() => toSQL({} as unknown as IntoSql)).toThrow(
+    "Unsupported value type: object value: ([object Object])",
+  );
+});
+test("toSQL() throws on unsupported value type", () => {
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  expect(() => (<any>toSQL)()).toThrow(
+    "Unsupported value type: undefined value: (undefined)",
+  );
+});

nodejs/biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/1.7.3/schema.json",
+  "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
   "organizeImports": {
     "enabled": true
   },
@@ -77,7 +77,7 @@
         "noDuplicateObjectKeys": "error",
         "noDuplicateParameters": "error",
         "noEmptyBlockStatements": "error",
-        "noExplicitAny": "error",
+        "noExplicitAny": "warn",
         "noExtraNonNullAssertion": "error",
         "noFallthroughSwitchClause": "error",
         "noFunctionAssign": "error",
@@ -94,12 +94,28 @@
         "useValidTypeof": "error"
       }
     },
-    "ignore": ["**/dist/**/*", "**/native.js", "**/native.d.ts"]
+    "ignore": [
+      "**/dist/**/*",
+      "**/native.js",
+      "**/native.d.ts",
+      "__test__/docs/**/*",
+      "examples/**/*"
+    ]
   },
   "javascript": {
     "globals": []
   },
   "overrides": [
+    {
+      "include": ["__test__/s3_integration.test.ts"],
+      "linter": {
+        "rules": {
+          "style": {
+            "useNamingConvention": "off"
+          }
+        }
+      }
+    },
     {
       "include": [
         "**/*.ts",

nodejs/examples/.gitignore

@@ -0,0 +1 @@
+data/

nodejs/examples/ann_indexes.ts

@@ -0,0 +1,49 @@
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+
+// --8<-- [start:ingest]
+const db = await lancedb.connect("/tmp/lancedb/");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: `${i}`,
+  content: "",
+  longId: `${i}`,
+}));
+
+const table = await db.createTable("my_vectors", data, { mode: "overwrite" });
+await table.createIndex("vector", {
+  config: lancedb.Index.ivfPq({
+    numPartitions: 16,
+    numSubVectors: 48,
+  }),
+});
+// --8<-- [end:ingest]
+
+// --8<-- [start:search1]
+const _results1 = await table
+  .search(Array(1536).fill(1.2))
+  .limit(2)
+  .nprobes(20)
+  .refineFactor(10)
+  .toArray();
+// --8<-- [end:search1]
+
+// --8<-- [start:search2]
+const _results2 = await table
+  .search(Array(1536).fill(1.2))
+  .where("id != '1141'")
+  .limit(2)
+  .toArray();
+// --8<-- [end:search2]
+
+// --8<-- [start:search3]
+const _results3 = await table
+  .search(Array(1536).fill(1.2))
+  .select(["id"])
+  .limit(2)
+  .toArray();
+// --8<-- [end:search3]
+
+console.log("Ann indexes: done");

nodejs/examples/basic.ts

@@ -0,0 +1,149 @@
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import * as arrow from "apache-arrow";
+import { Field, FixedSizeList, Float16, Int32, Schema } from "apache-arrow";
+
+// --8<-- [end:imports]
+
+// --8<-- [start:connect]
+const uri = "/tmp/lancedb/";
+const db = await lancedb.connect(uri);
+// --8<-- [end:connect]
+{
+  // --8<-- [start:create_table]
+  const data = [
+    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+  ];
+  const _tbl = await db.createTable("myTable", data);
+  // --8<-- [end:create_table]
+  {
+    // --8<-- [start:create_table_exists_ok]
+    const _tbl = await db.createTable("myTable", data, {
+      existsOk: true,
+    });
+    // --8<-- [end:create_table_exists_ok]
+  }
+  {
+    // --8<-- [start:create_table_overwrite]
+    const _tbl = await db.createTable("myTable", data, {
+      mode: "overwrite",
+    });
+    // --8<-- [end:create_table_overwrite]
+  }
+}
+
+{
+  // --8<-- [start:create_table_with_schema]
+  const schema = new arrow.Schema([
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+    ),
+    new arrow.Field("item", new arrow.Utf8(), true),
+    new arrow.Field("price", new arrow.Float32(), true),
+  ]);
+  const data = [
+    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+  ];
+  const _tbl = await db.createTable("myTable", data, {
+    schema,
+  });
+  // --8<-- [end:create_table_with_schema]
+}
+
+{
+  // --8<-- [start:create_empty_table]
+  const schema = new arrow.Schema([
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+    ),
+  ]);
+  const _tbl = await db.createEmptyTable("empty_table", schema);
+  // --8<-- [end:create_empty_table]
+}
+{
+  // --8<-- [start:open_table]
+  const _tbl = await db.openTable("myTable");
+  // --8<-- [end:open_table]
+}
+
+{
+  // --8<-- [start:table_names]
+  const tableNames = await db.tableNames();
+  console.log(tableNames);
+  // --8<-- [end:table_names]
+}
+
+const tbl = await db.openTable("myTable");
+{
+  // --8<-- [start:add_data]
+  const data = [
+    { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
+    { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
+  ];
+  await tbl.add(data);
+  // --8<-- [end:add_data]
+}
+{
+  // --8<-- [start:vector_search]
+  const _res = tbl.search([100, 100]).limit(2).toArray();
+  // --8<-- [end:vector_search]
+}
+{
+  const data = Array.from({ length: 1000 })
+    .fill(null)
+    .map(() => ({
+      vector: [Math.random(), Math.random()],
+      item: "autogen",
+      price: Math.round(Math.random() * 100),
+    }));
+
+  await tbl.add(data);
+}
+
+// --8<-- [start:create_index]
+await tbl.createIndex("vector");
+// --8<-- [end:create_index]
+
+// --8<-- [start:delete_rows]
+await tbl.delete('item = "fizz"');
+// --8<-- [end:delete_rows]
+
+// --8<-- [start:drop_table]
+await db.dropTable("myTable");
+// --8<-- [end:drop_table]
+await db.dropTable("empty_table");
+
+{
+  // --8<-- [start:create_f16_table]
+  const db = await lancedb.connect("/tmp/lancedb");
+  const dim = 16;
+  const total = 10;
+  const f16Schema = new Schema([
+    new Field("id", new Int32()),
+    new Field(
+      "vector",
+      new FixedSizeList(dim, new Field("item", new Float16(), true)),
+      false,
+    ),
+  ]);
+  const data = lancedb.makeArrowTable(
+    Array.from(Array(total), (_, i) => ({
+      id: i,
+      vector: Array.from(Array(dim), Math.random),
+    })),
+    { schema: f16Schema },
+  );
+  const _table = await db.createTable("f16_tbl", data);
+  // --8<-- [end:create_f16_table]
+  await db.dropTable("f16_tbl");
+}

nodejs/examples/embedding.ts

@@ -0,0 +1,83 @@
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { type Float, Float32, Utf8 } from "apache-arrow";
+// --8<-- [end:imports]
+
+{
+  // --8<-- [start:openai_embeddings]
+
+  const db = await lancedb.connect("/tmp/db");
+  const func = getRegistry()
+    .get("openai")
+    ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+  const wordsSchema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+  const tbl = await db.createEmptyTable("words", wordsSchema, {
+    mode: "overwrite",
+  });
+  await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+  const query = "greetings";
+  const actual = (await (await tbl.search(query)).limit(1).toArray())[0];
+
+  // --8<-- [end:openai_embeddings]
+  console.log("result = ", actual.text);
+}
+
+{
+  // --8<-- [start:embedding_function]
+  const db = await lancedb.connect("/tmp/db");
+
+  @register("my_embedding")
+  class MyEmbeddingFunction extends EmbeddingFunction<string> {
+    toJSON(): object {
+      return {};
+    }
+    ndims() {
+      return 3;
+    }
+    embeddingDataType(): Float {
+      return new Float32();
+    }
+    async computeQueryEmbeddings(_data: string) {
+      // This is a placeholder for a real embedding function
+      return [1, 2, 3];
+    }
+    async computeSourceEmbeddings(data: string[]) {
+      // This is a placeholder for a real embedding function
+      return Array.from({ length: data.length }).fill([1, 2, 3]) as number[][];
+    }
+  }
+
+  const func = new MyEmbeddingFunction();
+
+  const data = [{ text: "pepperoni" }, { text: "pineapple" }];
+
+  // Option 1: manually specify the embedding function
+  const table = await db.createTable("vectors", data, {
+    embeddingFunction: {
+      function: func,
+      sourceColumn: "text",
+      vectorColumn: "vector",
+    },
+    mode: "overwrite",
+  });
+
+  // Option 2: provide the embedding function through a schema
+
+  const schema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+
+  const table2 = await db.createTable("vectors2", data, {
+    schema,
+    mode: "overwrite",
+  });
+  // --8<-- [end:embedding_function]
+}

nodejs/examples/filtering.ts

@@ -0,0 +1,34 @@
+import * as lancedb from "@lancedb/lancedb";
+
+const db = await lancedb.connect("data/sample-lancedb");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: i,
+  item: `item ${i}`,
+  strId: `${i}`,
+}));
+
+const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+// --8<-- [start:search]
+const _result = await tbl
+  .search(Array(1536).fill(0.5))
+  .limit(1)
+  .where("id = 10")
+  .toArray();
+// --8<-- [end:search]
+
+// --8<-- [start:vec_search]
+await tbl
+  .search(Array(1536).fill(0))
+  .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+  .postfilter()
+  .toArray();
+// --8<-- [end:vec_search]
+
+// --8<-- [start:sql_search]
+await tbl.query().where("id = 10").limit(10).toArray();
+// --8<-- [end:sql_search]
+
+console.log("SQL search: done");

nodejs/examples/jsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

nodejs/examples/package-lock.json

@@ -0,0 +1,79 @@
+{
+  "name": "examples",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "examples",
+      "version": "1.0.0",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@lancedb/lancedb": "file:../"
+      },
+      "peerDependencies": {
+        "typescript": "^5.0.0"
+      }
+    },
+    "..": {
+      "name": "@lancedb/lancedb",
+      "version": "0.6.0",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "apache-arrow": "^15.0.0",
+        "axios": "^1.7.2",
+        "openai": "^4.29.2",
+        "reflect-metadata": "^0.2.2"
+      },
+      "devDependencies": {
+        "@aws-sdk/client-kms": "^3.33.0",
+        "@aws-sdk/client-s3": "^3.33.0",
+        "@biomejs/biome": "^1.7.3",
+        "@jest/globals": "^29.7.0",
+        "@napi-rs/cli": "^2.18.0",
+        "@types/axios": "^0.14.0",
+        "@types/jest": "^29.1.2",
+        "@types/tmp": "^0.2.6",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
+        "eslint": "^8.57.0",
+        "jest": "^29.7.0",
+        "shx": "^0.3.4",
+        "tmp": "^0.2.3",
+        "ts-jest": "^29.1.2",
+        "typedoc": "^0.25.7",
+        "typedoc-plugin-markdown": "^3.17.1",
+        "typescript": "^5.3.3",
+        "typescript-eslint": "^7.1.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb": {
+      "resolved": "..",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.5.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.2.tgz",
+      "integrity": "sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==",
+      "peer": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    }
+  }
+}

nodejs/examples/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "examples",
+  "version": "1.0.0",
+  "description": "Examples for LanceDB",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@lancedb/lancedb": "file:../"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  }
+}

nodejs/examples/search.ts

@@ -0,0 +1,37 @@
+// --8<-- [end:import]
+import * as fs from "node:fs";
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+
+async function setup() {
+  fs.rmSync("data/sample-lancedb", { recursive: true, force: true });
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  const data = Array.from({ length: 10_000 }, (_, i) => ({
+    vector: Array(1536).fill(i),
+    id: `${i}`,
+    content: "",
+    longId: `${i}`,
+  }));
+
+  await db.createTable("my_vectors", data);
+}
+
+await setup();
+
+// --8<-- [start:search1]
+const db = await lancedb.connect("data/sample-lancedb");
+const tbl = await db.openTable("my_vectors");
+
+const _results1 = await tbl.search(Array(1536).fill(1.2)).limit(10).toArray();
+// --8<-- [end:search1]
+
+// --8<-- [start:search2]
+const _results2 = await tbl
+  .search(Array(1536).fill(1.2))
+  .distanceType("cosine")
+  .limit(10)
+  .toArray();
+// --8<-- [end:search2]
+
+console.log("search: done");

nodejs/lancedb/arrow.ts

@@ -15,6 +15,7 @@
 import {
   Table as ArrowTable,
   Binary,
+  BufferType,
   DataType,
   Field,
   FixedSizeBinary,
@@ -37,14 +38,72 @@ import {
   type makeTable,
   vectorFromArray,
 } from "apache-arrow";
+import { Buffers } from "apache-arrow/data";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
-import { sanitizeField, sanitizeSchema, sanitizeType } from "./sanitize";
+import {
+  sanitizeField,
+  sanitizeSchema,
+  sanitizeTable,
+  sanitizeType,
+} from "./sanitize";
 export * from "apache-arrow";
+export type SchemaLike =
+  | Schema
+  | {
+      fields: FieldLike[];
+      metadata: Map<string, string>;
+      get names(): unknown[];
+    };
+export type FieldLike =
+  | Field
+  | {
+      type: string;
+      name: string;
+      nullable?: boolean;
+      metadata?: Map<string, string>;
+    };
 
-export type IntoVector = Float32Array | Float64Array | number[];
+export type DataLike =
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  | import("apache-arrow").Data<Struct<any>>
+  | {
+      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+      type: any;
+      length: number;
+      offset: number;
+      stride: number;
+      nullable: boolean;
+      children: DataLike[];
+      get nullCount(): number;
+      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+      values: Buffers<any>[BufferType.DATA];
+      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+      typeIds: Buffers<any>[BufferType.TYPE];
+      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+      nullBitmap: Buffers<any>[BufferType.VALIDITY];
+      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+      valueOffsets: Buffers<any>[BufferType.OFFSET];
+    };
 
-export function isArrowTable(value: object): value is ArrowTable {
+export type RecordBatchLike =
+  | RecordBatch
+  | {
+      schema: SchemaLike;
+      data: DataLike;
+    };
+
+export type TableLike =
+  | ArrowTable
+  | { schema: SchemaLike; batches: RecordBatchLike[] };
+
+export type IntoVector =
+  | Float32Array
+  | Float64Array
+  | number[]
+  | Promise<Float32Array | Float64Array | number[]>;
+
+export function isArrowTable(value: object): value is TableLike {
   if (value instanceof ArrowTable) return true;
   return "schema" in value && "batches" in value;
 }
@@ -135,7 +194,7 @@ export function isFixedSizeList(value: unknown): value is FixedSizeList {
 }
 
 /** Data type accepted by NodeJS SDK */
-export type Data = Record<string, unknown>[] | ArrowTable;
+export type Data = Record<string, unknown>[] | TableLike;
 
 /*
  * Options to control how a column should be converted to a vector array
@@ -162,7 +221,7 @@ export class MakeArrowTableOptions {
    * The schema must be specified if there are no records (e.g. to make
    * an empty table)
    */
-  schema?: Schema;
+  schema?: SchemaLike;
 
   /*
    * Mapping from vector column name to expected type
@@ -310,7 +369,7 @@ export function makeArrowTable(
   if (opt.schema !== undefined && opt.schema !== null) {
     opt.schema = sanitizeSchema(opt.schema);
     opt.schema = validateSchemaEmbeddings(
-      opt.schema,
+      opt.schema as Schema,
       data,
       options?.embeddingFunction,
     );
@@ -394,7 +453,7 @@ export function makeArrowTable(
     // `new ArrowTable(schema, batches)` which does not do any schema inference
     const firstTable = new ArrowTable(columns);
     const batchesFixed = firstTable.batches.map(
-      (batch) => new RecordBatch(opt.schema!, batch.data),
+      (batch) => new RecordBatch(opt.schema as Schema, batch.data),
     );
     let schema: Schema;
     if (metadata !== undefined) {
@@ -407,9 +466,9 @@ export function makeArrowTable(
         }
       }
 
-      schema = new Schema(opt.schema.fields, schemaMetadata);
+      schema = new Schema(opt.schema.fields as Field[], schemaMetadata);
     } else {
-      schema = opt.schema;
+      schema = opt.schema as Schema;
     }
     return new ArrowTable(schema, batchesFixed);
   }
@@ -425,7 +484,7 @@ export function makeArrowTable(
  * Create an empty Arrow table with the provided schema
  */
 export function makeEmptyTable(
-  schema: Schema,
+  schema: SchemaLike,
   metadata?: Map<string, string>,
 ): ArrowTable {
   return makeArrowTable([], { schema }, metadata);
@@ -563,17 +622,16 @@ async function applyEmbeddingsFromMetadata(
 async function applyEmbeddings<T>(
   table: ArrowTable,
   embeddings?: EmbeddingFunctionConfig,
-  schema?: Schema,
+  schema?: SchemaLike,
 ): Promise<ArrowTable> {
-  if (schema?.metadata.has("embedding_functions")) {
-    return applyEmbeddingsFromMetadata(table, schema!);
-  } else if (embeddings == null || embeddings === undefined) {
-    return table;
-  }
-
   if (schema !== undefined && schema !== null) {
     schema = sanitizeSchema(schema);
   }
+  if (schema?.metadata.has("embedding_functions")) {
+    return applyEmbeddingsFromMetadata(table, schema! as Schema);
+  } else if (embeddings == null || embeddings === undefined) {
+    return table;
+  }
 
   // Convert from ArrowTable to Record<String, Vector>
   const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
@@ -650,7 +708,7 @@ async function applyEmbeddings<T>(
         `When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`,
       );
     }
-    return alignTable(newTable, schema);
+    return alignTable(newTable, schema as Schema);
   }
   return newTable;
 }
@@ -744,7 +802,7 @@ export async function fromRecordsToStreamBuffer(
 export async function fromTableToBuffer(
   table: ArrowTable,
   embeddings?: EmbeddingFunctionConfig,
-  schema?: Schema,
+  schema?: SchemaLike,
 ): Promise<Buffer> {
   if (schema !== undefined && schema !== null) {
     schema = sanitizeSchema(schema);
@@ -771,7 +829,7 @@ export async function fromDataToBuffer(
     schema = sanitizeSchema(schema);
   }
   if (isArrowTable(data)) {
-    return fromTableToBuffer(data, embeddings, schema);
+    return fromTableToBuffer(sanitizeTable(data), embeddings, schema);
   } else {
     const table = await convertToTable(data, embeddings, { schema });
     return fromTableToBuffer(table);
@@ -789,7 +847,7 @@ export async function fromDataToBuffer(
 export async function fromTableToStreamBuffer(
   table: ArrowTable,
   embeddings?: EmbeddingFunctionConfig,
-  schema?: Schema,
+  schema?: SchemaLike,
 ): Promise<Buffer> {
   const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
   const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
@@ -854,7 +912,6 @@ function validateSchemaEmbeddings(
   for (let field of schema.fields) {
     if (isFixedSizeList(field.type)) {
       field = sanitizeField(field);
-
       if (data.length !== 0 && data?.[0]?.[field.name] === undefined) {
         if (schema.metadata.has("embedding_functions")) {
           const embeddings = JSON.parse(

nodejs/lancedb/connection.ts

@@ -12,38 +12,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Table as ArrowTable, Schema } from "./arrow";
+import { Data, Schema, SchemaLike, TableLike } from "./arrow";
-import {
+import { fromTableToBuffer, makeEmptyTable } from "./arrow";
-  fromTableToBuffer,
-  isArrowTable,
-  makeArrowTable,
-  makeEmptyTable,
-} from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
-import { ConnectionOptions, Connection as LanceDbConnection } from "./native";
+import { Connection as LanceDbConnection } from "./native";
-import { Table } from "./table";
+import { LocalTable, Table } from "./table";
-
-/**
- * Connect to a LanceDB instance at the given URI.
- *
- * Accepted formats:
- *
- * - `/path/to/database` - local database
- * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
- * - `db://host:port` - remote database (LanceDB cloud)
- * @param {string} uri - The uri of the database. If the database uri starts
- * with `db://` then it connects to a remote database.
- * @see {@link ConnectionOptions} for more details on the URI format.
- */
-export async function connect(
-  uri: string,
-  opts?: Partial<ConnectionOptions>,
-): Promise<Connection> {
-  opts = opts ?? {};
-  opts.storageOptions = cleanseStorageOptions(opts.storageOptions);
-  const nativeConn = await LanceDbConnection.new(uri, opts);
-  return new Connection(nativeConn);
-}
 
 export interface CreateTableOptions {
   /**
@@ -77,7 +50,7 @@ export interface CreateTableOptions {
    * The default is true while the new format is in beta
    */
   useLegacyFormat?: boolean;
-  schema?: Schema;
+  schema?: SchemaLike;
   embeddingFunction?: EmbeddingFunctionConfig;
 }
 
@@ -117,7 +90,6 @@ export interface TableNamesOptions {
   /** An optional limit to the number of results to return. */
   limit?: number;
 }
-
 /**
  * A LanceDB Connection that allows you to open tables and create new ones.
  *
@@ -136,17 +108,15 @@ export interface TableNamesOptions {
  * Any created tables are independent and will continue to work even if
  * the underlying connection has been closed.
  */
-export class Connection {
+export abstract class Connection {
-  readonly inner: LanceDbConnection;
+  [Symbol.for("nodejs.util.inspect.custom")](): string {
-
+    return this.display();
-  constructor(inner: LanceDbConnection) {
-    this.inner = inner;
   }
 
-  /** Return true if the connection has not been closed */
+  /**
-  isOpen(): boolean {
+   * Return true if the connection has not been closed
-    return this.inner.isOpen();
+   */
-  }
+  abstract isOpen(): boolean;
 
   /**
    * Close the connection, releasing any underlying resources.
@@ -155,14 +125,12 @@ export class Connection {
    *
    * Any attempt to use the connection after it is closed will result in an error.
    */
-  close(): void {
+  abstract close(): void;
-    this.inner.close();
-  }
 
-  /** Return a brief description of the connection */
+  /**
-  display(): string {
+   * Return a brief description of the connection
-    return this.inner.display();
+   */
-  }
+  abstract display(): string;
 
   /**
    * List all the table names in this database.
@@ -170,15 +138,86 @@ export class Connection {
    * Tables will be returned in lexicographical order.
    * @param {Partial<TableNamesOptions>} options - options to control the
    * paging / start point
+   *
    */
-  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
+  abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
-    return this.inner.tableNames(options?.startAfter, options?.limit);
-  }
 
   /**
    * Open a table in the database.
    * @param {string} name - The name of the table
    */
+  abstract openTable(
+    name: string,
+    options?: Partial<OpenTableOptions>,
+  ): Promise<Table>;
+
+  /**
+   * Creates a new Table and initialize it with new data.
+   * @param {object} options - The options object.
+   * @param {string} options.name - The name of the table.
+   * @param {Data} options.data - Non-empty Array of Records to be inserted into the table
+   *
+   */
+  abstract createTable(
+    options: {
+      name: string;
+      data: Data;
+    } & Partial<CreateTableOptions>,
+  ): Promise<Table>;
+  /**
+   * Creates a new Table and initialize it with new data.
+   * @param {string} name - The name of the table.
+   * @param {Record<string, unknown>[] | TableLike} data - Non-empty Array of Records
+   * to be inserted into the table
+   */
+  abstract createTable(
+    name: string,
+    data: Record<string, unknown>[] | TableLike,
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table>;
+
+  /**
+   * Creates a new empty Table
+   * @param {string} name - The name of the table.
+   * @param {Schema} schema - The schema of the table
+   */
+  abstract createEmptyTable(
+    name: string,
+    schema: import("./arrow").SchemaLike,
+    options?: Partial<CreateTableOptions>,
+  ): Promise<Table>;
+
+  /**
+   * Drop an existing table.
+   * @param {string} name The name of the table to drop.
+   */
+  abstract dropTable(name: string): Promise<void>;
+}
+
+export class LocalConnection extends Connection {
+  readonly inner: LanceDbConnection;
+
+  constructor(inner: LanceDbConnection) {
+    super();
+    this.inner = inner;
+  }
+
+  isOpen(): boolean {
+    return this.inner.isOpen();
+  }
+
+  close(): void {
+    this.inner.close();
+  }
+
+  display(): string {
+    return this.inner.display();
+  }
+
+  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
+    return this.inner.tableNames(options?.startAfter, options?.limit);
+  }
+
   async openTable(
     name: string,
     options?: Partial<OpenTableOptions>,
@@ -189,58 +228,38 @@ export class Connection {
       options?.indexCacheSize,
     );
 
-    return new Table(innerTable);
+    return new LocalTable(innerTable);
   }
 
-  /**
-   * Creates a new Table and initialize it with new data.
-   * @param {string} name - The name of the table.
-   * @param {Record<string, unknown>[] | ArrowTable} data - Non-empty Array of Records
-   * to be inserted into the table
-   */
   async createTable(
-    name: string,
+    nameOrOptions:
-    data: Record<string, unknown>[] | ArrowTable,
+      | string
+      | ({ name: string; data: Data } & Partial<CreateTableOptions>),
+    data?: Record<string, unknown>[] | TableLike,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
-    let mode: string = options?.mode ?? "create";
+    if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
-    const existOk = options?.existOk ?? false;
+      const { name, data, ...options } = nameOrOptions;
-
+      return this.createTable(name, data, options);
-    if (mode === "create" && existOk) {
-      mode = "exist_ok";
     }
-
+    if (data === undefined) {
-    let table: ArrowTable;
+      throw new Error("data is required");
-    if (isArrowTable(data)) {
-      table = data;
-    } else {
-      table = makeArrowTable(data, options);
     }
-
+    const { buf, mode } = await Table.parseTableData(data, options);
-    const buf = await fromTableToBuffer(
-      table,
-      options?.embeddingFunction,
-      options?.schema,
-    );
     const innerTable = await this.inner.createTable(
-      name,
+      nameOrOptions,
       buf,
       mode,
       cleanseStorageOptions(options?.storageOptions),
       options?.useLegacyFormat,
     );
 
-    return new Table(innerTable);
+    return new LocalTable(innerTable);
   }
 
-  /**
-   * Creates a new empty Table
-   * @param {string} name - The name of the table.
-   * @param {Schema} schema - The schema of the table
-   */
   async createEmptyTable(
     name: string,
-    schema: Schema,
+    schema: import("./arrow").SchemaLike,
     options?: Partial<CreateTableOptions>,
   ): Promise<Table> {
     let mode: string = options?.mode ?? "create";
@@ -265,13 +284,9 @@ export class Connection {
       cleanseStorageOptions(options?.storageOptions),
       options?.useLegacyFormat,
     );
-    return new Table(innerTable);
+    return new LocalTable(innerTable);
   }
 
-  /**
-   * Drop an existing table.
-   * @param {string} name The name of the table to drop.
-   */
   async dropTable(name: string): Promise<void> {
     return this.inner.dropTable(name);
   }
@@ -280,7 +295,7 @@ export class Connection {
 /**
  * Takes storage options and makes all the keys snake case.
  */
-function cleanseStorageOptions(
+export function cleanseStorageOptions(
   options?: Record<string, string>,
 ): Record<string, string> | undefined {
   if (options === undefined) {

nodejs/lancedb/embedding/embedding_function.ts

@@ -35,6 +35,11 @@ export interface FunctionOptions {
   [key: string]: any;
 }
 
+export interface EmbeddingFunctionConstructor<
+  T extends EmbeddingFunction = EmbeddingFunction,
+> {
+  new (modelOptions?: T["TOptions"]): T;
+}
 /**
  * An embedding function that automatically creates vector representation for a given column.
  */
@@ -43,6 +48,12 @@ export abstract class EmbeddingFunction<
   T = any,
   M extends FunctionOptions = FunctionOptions,
 > {
+  /**
+   * @ignore
+   *  This is only used for associating the options type with the class for type checking
+   */
+  // biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
+  readonly TOptions!: M;
   /**
    * Convert the embedding function to a JSON object
    * It is used to serialize the embedding function to the schema
@@ -170,7 +181,7 @@ export abstract class EmbeddingFunction<
   /**
   Compute the embeddings for a single query
  */
-  async computeQueryEmbeddings(data: T): Promise<IntoVector> {
+  async computeQueryEmbeddings(data: T): Promise<Awaited<IntoVector>> {
     return this.computeSourceEmbeddings([data]).then(
       (embeddings) => embeddings[0],
     );

nodejs/lancedb/embedding/openai.ts

@@ -13,24 +13,29 @@
 // limitations under the License.
 
 import type OpenAI from "openai";
+import { type EmbeddingCreateParams } from "openai/resources";
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
 import { register } from "./registry";
 
 export type OpenAIOptions = {
-  apiKey?: string;
+  apiKey: string;
-  model?: string;
+  model: EmbeddingCreateParams["model"];
 };
 
 @register("openai")
 export class OpenAIEmbeddingFunction extends EmbeddingFunction<
   string,
-  OpenAIOptions
+  Partial<OpenAIOptions>
 > {
   #openai: OpenAI;
-  #modelName: string;
+  #modelName: OpenAIOptions["model"];
 
-  constructor(options: OpenAIOptions = { model: "text-embedding-ada-002" }) {
+  constructor(
+    options: Partial<OpenAIOptions> = {
+      model: "text-embedding-ada-002",
+    },
+  ) {
     super();
     const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
     if (!openAIKey) {
@@ -73,7 +78,7 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
       case "text-embedding-3-small":
         return 1536;
       default:
-        return null as never;
+        throw new Error(`Unknown model: ${this.#modelName}`);
     }
   }
 

nodejs/lancedb/embedding/registry.ts

@@ -12,21 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import type { EmbeddingFunction } from "./embedding_function";
+import {
+  type EmbeddingFunction,
+  type EmbeddingFunctionConstructor,
+} from "./embedding_function";
 import "reflect-metadata";
-
+import { OpenAIEmbeddingFunction } from "./openai";
-export interface EmbeddingFunctionOptions {
-  [key: string]: unknown;
-}
-
-export interface EmbeddingFunctionFactory<
-  T extends EmbeddingFunction = EmbeddingFunction,
-> {
-  new (modelOptions?: EmbeddingFunctionOptions): T;
-}
 
 interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
-  create(options?: EmbeddingFunctionOptions): T;
+  create(options?: T["TOptions"]): T;
 }
 
 /**
@@ -36,7 +30,7 @@ interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
  * or TextEmbeddingFunction and registering it with the registry
  */
 export class EmbeddingFunctionRegistry {
-  #functions: Map<string, EmbeddingFunctionFactory> = new Map();
+  #functions = new Map<string, EmbeddingFunctionConstructor>();
 
   /**
    * Register an embedding function
@@ -44,7 +38,9 @@ export class EmbeddingFunctionRegistry {
    * @param func The function to register
    * @throws Error if the function is already registered
    */
-  register<T extends EmbeddingFunctionFactory = EmbeddingFunctionFactory>(
+  register<
+    T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
+  >(
     this: EmbeddingFunctionRegistry,
     alias?: string,
     // biome-ignore lint/suspicious/noExplicitAny: <explanation>
@@ -69,18 +65,34 @@ export class EmbeddingFunctionRegistry {
    * Fetch an embedding function by name
    * @param name The name of the function
    */
-  get<T extends EmbeddingFunction<unknown> = EmbeddingFunction>(
+  get<T extends EmbeddingFunction<unknown>, Name extends string = "">(
-    name: string,
+    name: Name extends "openai" ? "openai" : string,
-  ): EmbeddingFunctionCreate<T> | undefined {
+    //This makes it so that you can use string constants as "types", or use an explicitly supplied type
+    // ex:
+    // `registry.get("openai") -> EmbeddingFunctionCreate<OpenAIEmbeddingFunction>`
+    // `registry.get<MyCustomEmbeddingFunction>("my_func") -> EmbeddingFunctionCreate<MyCustomEmbeddingFunction> | undefined`
+    //
+    // the reason this is important is that we always know our built in functions are defined so the user isnt forced to do a non null/undefined
+    // ```ts
+    // const openai: OpenAIEmbeddingFunction = registry.get("openai").create()
+    // ```
+  ): Name extends "openai"
+    ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction>
+    : EmbeddingFunctionCreate<T> | undefined {
+    type Output = Name extends "openai"
+      ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction>
+      : EmbeddingFunctionCreate<T> | undefined;
+
     const factory = this.#functions.get(name);
     if (!factory) {
-      return undefined;
+      return undefined as Output;
     }
+
     return {
-      create: function (options: EmbeddingFunctionOptions) {
+      create: function (options?: T["TOptions"]) {
-        return new factory(options) as unknown as T;
+        return new factory(options);
       },
-    };
+    } as Output;
   }
 
   /**
@@ -104,7 +116,7 @@ export class EmbeddingFunctionRegistry {
         name: string;
         sourceColumn: string;
         vectorColumn: string;
-        model: EmbeddingFunctionOptions;
+        model: EmbeddingFunction["TOptions"];
       };
       const functions = <FunctionConfig[]>(
         JSON.parse(metadata.get("embedding_functions")!)

nodejs/lancedb/index.ts

@@ -12,25 +12,43 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import {
+  Connection,
+  LocalConnection,
+  cleanseStorageOptions,
+} from "./connection";
+
+import {
+  ConnectionOptions,
+  Connection as LanceDbConnection,
+} from "./native.js";
+
+import { RemoteConnection, RemoteConnectionOptions } from "./remote";
+
 export {
   WriteOptions,
   WriteMode,
   AddColumnsSql,
   ColumnAlteration,
   ConnectionOptions,
+  IndexStatistics,
+  IndexMetadata,
+  IndexConfig,
 } from "./native.js";
+
 export {
   makeArrowTable,
   MakeArrowTableOptions,
   Data,
   VectorColumnOptions,
 } from "./arrow";
+
 export {
-  connect,
   Connection,
   CreateTableOptions,
   TableNamesOptions,
 } from "./connection";
+
 export {
   ExecutableQuery,
   Query,
@@ -38,6 +56,87 @@ export {
   VectorQuery,
   RecordBatchIterator,
 } from "./query";
+
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
-export { Table, AddDataOptions, IndexConfig, UpdateOptions } from "./table";
+
+export { Table, AddDataOptions, UpdateOptions } from "./table";
+
 export * as embedding from "./embedding";
+
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect("/path/to/database");
+ * ```
+ * @example
+ * ```ts
+ * const conn = await connect(
+ *   "s3://bucket/path/to/database",
+ *   {storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ */
+export async function connect(
+  uri: string,
+  opts?: Partial<ConnectionOptions | RemoteConnectionOptions>,
+): Promise<Connection>;
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param  options - The options to use when connecting to the database
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect({
+ *   uri: "/path/to/database",
+ *   storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ */
+export async function connect(
+  opts: Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string },
+): Promise<Connection>;
+export async function connect(
+  uriOrOptions:
+    | string
+    | (Partial<RemoteConnectionOptions | ConnectionOptions> & { uri: string }),
+  opts: Partial<ConnectionOptions | RemoteConnectionOptions> = {},
+): Promise<Connection> {
+  let uri: string | undefined;
+  if (typeof uriOrOptions !== "string") {
+    const { uri: uri_, ...options } = uriOrOptions;
+    uri = uri_;
+    opts = options;
+  } else {
+    uri = uriOrOptions;
+  }
+
+  if (!uri) {
+    throw new Error("uri is required");
+  }
+
+  if (uri?.startsWith("db://")) {
+    return new RemoteConnection(uri, opts as RemoteConnectionOptions);
+  }
+  opts = (opts as ConnectionOptions) ?? {};
+  (<ConnectionOptions>opts).storageOptions = cleanseStorageOptions(
+    (<ConnectionOptions>opts).storageOptions,
+  );
+  const nativeConn = await LanceDbConnection.new(uri, opts);
+  return new LocalConnection(nativeConn);
+}

nodejs/lancedb/merge.ts

@@ -0,0 +1,70 @@
+import { Data, fromDataToBuffer } from "./arrow";
+import { NativeMergeInsertBuilder } from "./native";
+
+/** A builder used to create and run a merge insert operation */
+export class MergeInsertBuilder {
+  #native: NativeMergeInsertBuilder;
+
+  /** Construct a MergeInsertBuilder. __Internal use only.__ */
+  constructor(native: NativeMergeInsertBuilder) {
+    this.#native = native;
+  }
+
+  /**
+   * Rows that exist in both the source table (new data) and
+   * the target table (old data) will be updated, replacing
+   * the old row with the corresponding matching row.
+   *
+   * If there are multiple matches then the behavior is undefined.
+   * Currently this causes multiple copies of the row to be created
+   * but that behavior is subject to change.
+   *
+   * An optional condition may be specified.  If it is, then only
+   * matched rows that satisfy the condtion will be updated.  Any
+   * rows that do not satisfy the condition will be left as they
+   * are.  Failing to satisfy the condition does not cause a
+   * "matched row" to become a "not matched" row.
+   *
+   * The condition should be an SQL string.  Use the prefix
+   * target. to refer to rows in the target table (old data)
+   * and the prefix source. to refer to rows in the source
+   * table (new data).
+   *
+   * For example, "target.last_update < source.last_update"
+   */
+  whenMatchedUpdateAll(options?: { where: string }): MergeInsertBuilder {
+    return new MergeInsertBuilder(
+      this.#native.whenMatchedUpdateAll(options?.where),
+    );
+  }
+  /**
+   * Rows that exist only in the source table (new data) should
+   * be inserted into the target table.
+   */
+  whenNotMatchedInsertAll(): MergeInsertBuilder {
+    return new MergeInsertBuilder(this.#native.whenNotMatchedInsertAll());
+  }
+  /**
+   * Rows that exist only in the target table (old data) will be
+   * deleted.  An optional condition can be provided to limit what
+   * data is deleted.
+   *
+   * @param options.where - An optional condition to limit what data is deleted
+   */
+  whenNotMatchedBySourceDelete(options?: {
+    where: string;
+  }): MergeInsertBuilder {
+    return new MergeInsertBuilder(
+      this.#native.whenNotMatchedBySourceDelete(options?.where),
+    );
+  }
+  /**
+   * Executes the merge insert operation
+   *
+   * Nothing is returned but the `Table` is updated
+   */
+  async execute(data: Data): Promise<void> {
+    const buffer = await fromDataToBuffer(data);
+    await this.#native.execute(buffer);
+  }
+}

nodejs/lancedb/query.ts

@@ -89,15 +89,26 @@ export interface QueryExecutionOptions {
 }
 
 /** Common methods supported by all query types */
-export class QueryBase<
+export class QueryBase<NativeQueryType extends NativeQuery | NativeVectorQuery>
-  NativeQueryType extends NativeQuery | NativeVectorQuery,
+  implements AsyncIterable<RecordBatch>
-  QueryType,
-> implements AsyncIterable<RecordBatch>
 {
-  protected constructor(protected inner: NativeQueryType) {
+  protected constructor(
+    protected inner: NativeQueryType | Promise<NativeQueryType>,
+  ) {
     // intentionally empty
   }
 
+  // call a function on the inner (either a promise or the actual object)
+  protected doCall(fn: (inner: NativeQueryType) => void) {
+    if (this.inner instanceof Promise) {
+      this.inner = this.inner.then((inner) => {
+        fn(inner);
+        return inner;
+      });
+    } else {
+      fn(this.inner);
+    }
+  }
   /**
    * A filter statement to be applied to this query.
    *
@@ -110,9 +121,17 @@ export class QueryBase<
    * Filtering performance can often be improved by creating a scalar index
    * on the filter column(s).
    */
-  where(predicate: string): QueryType {
+  where(predicate: string): this {
-    this.inner.onlyIf(predicate);
+    this.doCall((inner: NativeQueryType) => inner.onlyIf(predicate));
-    return this as unknown as QueryType;
+    return this;
+  }
+  /**
+   * A filter statement to be applied to this query.
+   * @alias where
+   * @deprecated Use `where` instead
+   */
+  filter(predicate: string): this {
+    return this.where(predicate);
   }
 
   /**
@@ -147,7 +166,7 @@ export class QueryBase<
    */
   select(
     columns: string[] | Map<string, string> | Record<string, string> | string,
-  ): QueryType {
+  ): this {
     let columnTuples: [string, string][];
     if (typeof columns === "string") {
       columns = [columns];
@@ -159,8 +178,10 @@ export class QueryBase<
     } else {
       columnTuples = Object.entries(columns);
     }
-    this.inner.select(columnTuples);
+    this.doCall((inner: NativeQueryType) => {
-    return this as unknown as QueryType;
+      inner.select(columnTuples);
+    });
+    return this;
   }
 
   /**
@@ -169,15 +190,19 @@ export class QueryBase<
    * By default, a plain search has no limit.  If this method is not
    * called then every valid row from the table will be returned.
    */
-  limit(limit: number): QueryType {
+  limit(limit: number): this {
-    this.inner.limit(limit);
+    this.doCall((inner: NativeQueryType) => inner.limit(limit));
-    return this as unknown as QueryType;
+    return this;
   }
 
   protected nativeExecute(
     options?: Partial<QueryExecutionOptions>,
   ): Promise<NativeBatchIterator> {
-    return this.inner.execute(options?.maxBatchLength);
+    if (this.inner instanceof Promise) {
+      return this.inner.then((inner) => inner.execute(options?.maxBatchLength));
+    } else {
+      return this.inner.execute(options?.maxBatchLength);
+    }
   }
 
   /**
@@ -206,7 +231,13 @@ export class QueryBase<
   /** Collect the results as an Arrow @see {@link ArrowTable}. */
   async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
     const batches = [];
-    for await (const batch of new RecordBatchIterable(this.inner, options)) {
+    let inner;
+    if (this.inner instanceof Promise) {
+      inner = await this.inner;
+    } else {
+      inner = this.inner;
+    }
+    for await (const batch of new RecordBatchIterable(inner, options)) {
       batches.push(batch);
     }
     return new ArrowTable(batches);
@@ -218,6 +249,28 @@ export class QueryBase<
     const tbl = await this.toArrow(options);
     return tbl.toArray();
   }
+
+  /**
+   * Generates an explanation of the query execution plan.
+   *
+   * @example
+   * import * as lancedb from "@lancedb/lancedb"
+   * const db = await lancedb.connect("./.lancedb");
+   * const table = await db.createTable("my_table", [
+   *   { vector: [1.1, 0.9], id: "1" },
+   * ]);
+   * const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+   *
+   * @param verbose - If true, provides a more detailed explanation. Defaults to false.
+   * @returns A Promise that resolves to a string containing the query execution plan explanation.
+   */
+  async explainPlan(verbose = false): Promise<string> {
+    if (this.inner instanceof Promise) {
+      return this.inner.then((inner) => inner.explainPlan(verbose));
+    } else {
+      return this.inner.explainPlan(verbose);
+    }
+  }
 }
 
 /**
@@ -232,8 +285,8 @@ export interface ExecutableQuery {}
  *
  * This builder can be reused to execute the query many times.
  */
-export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
+export class VectorQuery extends QueryBase<NativeVectorQuery> {
-  constructor(inner: NativeVectorQuery) {
+  constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>) {
     super(inner);
   }
 
@@ -260,7 +313,8 @@ export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
    * you the desired recall.
    */
   nprobes(nprobes: number): VectorQuery {
-    this.inner.nprobes(nprobes);
+    super.doCall((inner) => inner.nprobes(nprobes));
+
     return this;
   }
 
@@ -274,7 +328,7 @@ export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
    * whose data type is a fixed-size-list of floats.
    */
   column(column: string): VectorQuery {
-    this.inner.column(column);
+    super.doCall((inner) => inner.column(column));
     return this;
   }
 
@@ -292,8 +346,10 @@ export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
    *
    * By default "l2" is used.
    */
-  distanceType(distanceType: string): VectorQuery {
+  distanceType(
-    this.inner.distanceType(distanceType);
+    distanceType: Required<IvfPqOptions>["distanceType"],
+  ): VectorQuery {
+    super.doCall((inner) => inner.distanceType(distanceType));
     return this;
   }
 
@@ -327,7 +383,7 @@ export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
    * distance between the query vector and the actual uncompressed vector.
    */
   refineFactor(refineFactor: number): VectorQuery {
-    this.inner.refineFactor(refineFactor);
+    super.doCall((inner) => inner.refineFactor(refineFactor));
     return this;
   }
 
@@ -352,7 +408,7 @@ export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
    * factor can often help restore some of the results lost by post filtering.
    */
   postfilter(): VectorQuery {
-    this.inner.postfilter();
+    super.doCall((inner) => inner.postfilter());
     return this;
   }
 
@@ -366,13 +422,13 @@ export class VectorQuery extends QueryBase<NativeVectorQuery, VectorQuery> {
    * calculate your recall to select an appropriate value for nprobes.
    */
   bypassVectorIndex(): VectorQuery {
-    this.inner.bypassVectorIndex();
+    super.doCall((inner) => inner.bypassVectorIndex());
     return this;
   }
 }
 
 /** A builder for LanceDB queries. */
-export class Query extends QueryBase<NativeQuery, Query> {
+export class Query extends QueryBase<NativeQuery> {
   constructor(tbl: NativeTable) {
     super(tbl.query());
   }
@@ -415,7 +471,37 @@ export class Query extends QueryBase<NativeQuery, Query> {
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
   nearestTo(vector: IntoVector): VectorQuery {
-    const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
+    if (this.inner instanceof Promise) {
-    return new VectorQuery(vectorQuery);
+      const nativeQuery = this.inner.then(async (inner) => {
+        if (vector instanceof Promise) {
+          const arr = await vector.then((v) => Float32Array.from(v));
+          return inner.nearestTo(arr);
+        } else {
+          return inner.nearestTo(Float32Array.from(vector));
+        }
+      });
+      return new VectorQuery(nativeQuery);
+    }
+    if (vector instanceof Promise) {
+      const res = (async () => {
+        try {
+          const v = await vector;
+          const arr = Float32Array.from(v);
+          //
+          // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
+          const value: any = this.nearestTo(arr);
+          const inner = value.inner as
+            | NativeVectorQuery
+            | Promise<NativeVectorQuery>;
+          return inner;
+        } catch (e) {
+          return Promise.reject(e);
+        }
+      })();
+      return new VectorQuery(res);
+    } else {
+      const vectorQuery = this.inner.nearestTo(Float32Array.from(vector));
+      return new VectorQuery(vectorQuery);
+    }
   }
 }

nodejs/lancedb/remote/client.ts

@@ -0,0 +1,221 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import axios, {
+  AxiosError,
+  type AxiosResponse,
+  type ResponseType,
+} from "axios";
+import { Table as ArrowTable } from "../arrow";
+import { tableFromIPC } from "../arrow";
+import { VectorQuery } from "../query";
+
+export class RestfulLanceDBClient {
+  #dbName: string;
+  #region: string;
+  #apiKey: string;
+  #hostOverride?: string;
+  #closed: boolean = false;
+  #connectionTimeout: number = 12 * 1000; // 12 seconds;
+  #readTimeout: number = 30 * 1000; // 30 seconds;
+  #session?: import("axios").AxiosInstance;
+
+  constructor(
+    dbName: string,
+    apiKey: string,
+    region: string,
+    hostOverride?: string,
+    connectionTimeout?: number,
+    readTimeout?: number,
+  ) {
+    this.#dbName = dbName;
+    this.#apiKey = apiKey;
+    this.#region = region;
+    this.#hostOverride = hostOverride ?? this.#hostOverride;
+    this.#connectionTimeout = connectionTimeout ?? this.#connectionTimeout;
+    this.#readTimeout = readTimeout ?? this.#readTimeout;
+  }
+
+  // todo: cache the session.
+  get session(): import("axios").AxiosInstance {
+    if (this.#session !== undefined) {
+      return this.#session;
+    } else {
+      return axios.create({
+        baseURL: this.url,
+        headers: {
+          // biome-ignore lint: external API
+          Authorization: `Bearer ${this.#apiKey}`,
+        },
+        transformResponse: decodeErrorData,
+        timeout: this.#connectionTimeout,
+      });
+    }
+  }
+
+  get url(): string {
+    return (
+      this.#hostOverride ??
+      `https://${this.#dbName}.${this.#region}.api.lancedb.com`
+    );
+  }
+
+  get headers(): { [key: string]: string } {
+    const headers: { [key: string]: string } = {
+      "x-api-key": this.#apiKey,
+      "x-request-id": "na",
+    };
+    if (this.#region == "local") {
+      headers["Host"] = `${this.#dbName}.${this.#region}.api.lancedb.com`;
+    }
+    if (this.#hostOverride) {
+      headers["x-lancedb-database"] = this.#dbName;
+    }
+    return headers;
+  }
+
+  isOpen(): boolean {
+    return !this.#closed;
+  }
+
+  private checkNotClosed(): void {
+    if (this.#closed) {
+      throw new Error("Connection is closed");
+    }
+  }
+
+  close(): void {
+    this.#session = undefined;
+    this.#closed = true;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  async get(uri: string, params?: Record<string, any>): Promise<any> {
+    this.checkNotClosed();
+    uri = new URL(uri, this.url).toString();
+    let response;
+    try {
+      response = await this.session.get(uri, {
+        headers: this.headers,
+        params,
+      });
+    } catch (e) {
+      if (e instanceof AxiosError) {
+        response = e.response;
+      } else {
+        throw e;
+      }
+    }
+
+    RestfulLanceDBClient.checkStatus(response!);
+    return response!.data;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: api response
+  async post(uri: string, body?: any): Promise<any>;
+  async post(
+    uri: string,
+    // biome-ignore lint/suspicious/noExplicitAny: api request
+    body: any,
+    additional: {
+      config?: { responseType: "arraybuffer" };
+      headers?: Record<string, string>;
+      params?: Record<string, string>;
+    },
+  ): Promise<Buffer>;
+  async post(
+    uri: string,
+    // biome-ignore lint/suspicious/noExplicitAny: api request
+    body?: any,
+    additional?: {
+      config?: { responseType: ResponseType };
+      headers?: Record<string, string>;
+      params?: Record<string, string>;
+    },
+    // biome-ignore lint/suspicious/noExplicitAny: api response
+  ): Promise<any> {
+    this.checkNotClosed();
+    uri = new URL(uri, this.url).toString();
+    additional = Object.assign(
+      { config: { responseType: "json" } },
+      additional,
+    );
+
+    const headers = { ...this.headers, ...additional.headers };
+
+    if (!headers["Content-Type"]) {
+      headers["Content-Type"] = "application/json";
+    }
+    let response;
+    try {
+      response = await this.session.post(uri, body, {
+        headers,
+        responseType: additional!.config!.responseType,
+        params: new Map(Object.entries(additional.params ?? {})),
+      });
+    } catch (e) {
+      if (e instanceof AxiosError) {
+        response = e.response;
+      } else {
+        throw e;
+      }
+    }
+    RestfulLanceDBClient.checkStatus(response!);
+    if (additional!.config!.responseType === "arraybuffer") {
+      return response!.data;
+    } else {
+      return JSON.parse(response!.data);
+    }
+  }
+
+  async listTables(limit = 10, pageToken = ""): Promise<string[]> {
+    const json = await this.get("/v1/table", { limit, pageToken });
+    return json.tables;
+  }
+
+  async query(tableName: string, query: VectorQuery): Promise<ArrowTable> {
+    const tbl = await this.post(`/v1/table/${tableName}/query`, query, {
+      config: {
+        responseType: "arraybuffer",
+      },
+    });
+    return tableFromIPC(tbl);
+  }
+
+  static checkStatus(response: AxiosResponse): void {
+    if (response.status === 404) {
+      throw new Error(`Not found: ${response.data}`);
+    } else if (response.status >= 400 && response.status < 500) {
+      throw new Error(
+        `Bad Request: ${response.status}, error: ${response.data}`,
+      );
+    } else if (response.status >= 500 && response.status < 600) {
+      throw new Error(
+        `Internal Server Error: ${response.status}, error: ${response.data}`,
+      );
+    } else if (response.status !== 200) {
+      throw new Error(
+        `Unknown Error: ${response.status}, error: ${response.data}`,
+      );
+    }
+  }
+}
+
+function decodeErrorData(data: unknown) {
+  if (Buffer.isBuffer(data)) {
+    const decoded = data.toString("utf-8");
+    return decoded;
+  }
+  return data;
+}

nodejs/lancedb/remote/connection.ts

@@ -0,0 +1,201 @@
+import { Schema } from "apache-arrow";
+import {
+  Data,
+  SchemaLike,
+  fromTableToStreamBuffer,
+  makeEmptyTable,
+} from "../arrow";
+import {
+  Connection,
+  CreateTableOptions,
+  OpenTableOptions,
+  TableNamesOptions,
+} from "../connection";
+import { Table } from "../table";
+import { TTLCache } from "../util";
+import { RestfulLanceDBClient } from "./client";
+import { RemoteTable } from "./table";
+
+export interface RemoteConnectionOptions {
+  apiKey?: string;
+  region?: string;
+  hostOverride?: string;
+  connectionTimeout?: number;
+  readTimeout?: number;
+}
+
+export class RemoteConnection extends Connection {
+  #dbName: string;
+  #apiKey: string;
+  #region: string;
+  #client: RestfulLanceDBClient;
+  #tableCache = new TTLCache(300_000);
+
+  constructor(
+    url: string,
+    {
+      apiKey,
+      region,
+      hostOverride,
+      connectionTimeout,
+      readTimeout,
+    }: RemoteConnectionOptions,
+  ) {
+    super();
+    apiKey = apiKey ?? process.env.LANCEDB_API_KEY;
+    region = region ?? process.env.LANCEDB_REGION;
+
+    if (!apiKey) {
+      throw new Error("apiKey is required when connecting to LanceDB Cloud");
+    }
+
+    if (!region) {
+      throw new Error("region is required when connecting to LanceDB Cloud");
+    }
+
+    const parsed = new URL(url);
+    if (parsed.protocol !== "db:") {
+      throw new Error(
+        `invalid protocol: ${parsed.protocol}, only accepts db://`,
+      );
+    }
+
+    this.#dbName = parsed.hostname;
+    this.#apiKey = apiKey;
+    this.#region = region;
+    this.#client = new RestfulLanceDBClient(
+      this.#dbName,
+      this.#apiKey,
+      this.#region,
+      hostOverride,
+      connectionTimeout,
+      readTimeout,
+    );
+  }
+
+  isOpen(): boolean {
+    return this.#client.isOpen();
+  }
+  close(): void {
+    return this.#client.close();
+  }
+
+  display(): string {
+    return `RemoteConnection(${this.#dbName})`;
+  }
+
+  async tableNames(options?: Partial<TableNamesOptions>): Promise<string[]> {
+    const response = await this.#client.get("/v1/table/", {
+      limit: options?.limit ?? 10,
+      // biome-ignore lint/style/useNamingConvention: <explanation>
+      page_token: options?.startAfter ?? "",
+    });
+    const body = await response.body();
+    for (const table of body.tables) {
+      this.#tableCache.set(table, true);
+    }
+    return body.tables;
+  }
+
+  async openTable(
+    name: string,
+    _options?: Partial<OpenTableOptions> | undefined,
+  ): Promise<Table> {
+    if (this.#tableCache.get(name) === undefined) {
+      await this.#client.post(
+        `/v1/table/${encodeURIComponent(name)}/describe/`,
+      );
+      this.#tableCache.set(name, true);
+    }
+    return new RemoteTable(this.#client, name, this.#dbName);
+  }
+
+  async createTable(
+    nameOrOptions:
+      | string
+      | ({ name: string; data: Data } & Partial<CreateTableOptions>),
+    data?: Data,
+    options?: Partial<CreateTableOptions> | undefined,
+  ): Promise<Table> {
+    if (typeof nameOrOptions !== "string" && "name" in nameOrOptions) {
+      const { name, data, ...options } = nameOrOptions;
+      return this.createTable(name, data, options);
+    }
+    if (data === undefined) {
+      throw new Error("data is required");
+    }
+    if (options?.mode) {
+      console.warn(
+        "option 'mode' is not supported in LanceDB Cloud",
+        "LanceDB Cloud only supports the default 'create' mode.",
+        "If the table already exists, an error will be thrown.",
+      );
+    }
+    if (options?.embeddingFunction) {
+      console.warn(
+        "embedding_functions is not yet supported on LanceDB Cloud.",
+        "Please vote https://github.com/lancedb/lancedb/issues/626 ",
+        "for this feature.",
+      );
+    }
+
+    const { buf } = await Table.parseTableData(
+      data,
+      options,
+      true /** streaming */,
+    );
+
+    await this.#client.post(
+      `/v1/table/${encodeURIComponent(nameOrOptions)}/create/`,
+      buf,
+      {
+        config: {
+          responseType: "arraybuffer",
+        },
+        headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+      },
+    );
+    this.#tableCache.set(nameOrOptions, true);
+    return new RemoteTable(this.#client, nameOrOptions, this.#dbName);
+  }
+
+  async createEmptyTable(
+    name: string,
+    schema: SchemaLike,
+    options?: Partial<CreateTableOptions> | undefined,
+  ): Promise<Table> {
+    if (options?.mode) {
+      console.warn(`mode is not supported on LanceDB Cloud`);
+    }
+
+    if (options?.embeddingFunction) {
+      console.warn(
+        "embeddingFunction is not yet supported on LanceDB Cloud.",
+        "Please vote https://github.com/lancedb/lancedb/issues/626 ",
+        "for this feature.",
+      );
+    }
+    const emptyTable = makeEmptyTable(schema);
+    const buf = await fromTableToStreamBuffer(emptyTable);
+
+    await this.#client.post(
+      `/v1/table/${encodeURIComponent(name)}/create/`,
+      buf,
+      {
+        config: {
+          responseType: "arraybuffer",
+        },
+        headers: { "Content-Type": "application/vnd.apache.arrow.stream" },
+      },
+    );
+
+    this.#tableCache.set(name, true);
+    return new RemoteTable(this.#client, name, this.#dbName);
+  }
+
+  async dropTable(name: string): Promise<void> {
+    await this.#client.post(`/v1/table/${encodeURIComponent(name)}/drop/`);
+
+    this.#tableCache.delete(name);
+  }
+}

nodejs/lancedb/remote/index.ts

@@ -0,0 +1,3 @@
+export { RestfulLanceDBClient } from "./client";
+export { type RemoteConnectionOptions, RemoteConnection } from "./connection";
+export { RemoteTable } from "./table";

nodejs/lancedb/remote/table.ts

@@ -0,0 +1,226 @@
+// Copyright 2023 LanceDB Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import { Table as ArrowTable } from "apache-arrow";
+
+import { Data, IntoVector } from "../arrow";
+
+import { IndexStatistics } from "..";
+import { CreateTableOptions } from "../connection";
+import { IndexOptions } from "../indices";
+import { MergeInsertBuilder } from "../merge";
+import { VectorQuery } from "../query";
+import { AddDataOptions, Table, UpdateOptions } from "../table";
+import { IntoSql, toSQL } from "../util";
+import { RestfulLanceDBClient } from "./client";
+
+export class RemoteTable extends Table {
+  #client: RestfulLanceDBClient;
+  #name: string;
+
+  // Used in the display() method
+  #dbName: string;
+
+  get #tablePrefix() {
+    return `/v1/table/${encodeURIComponent(this.#name)}/`;
+  }
+
+  get name(): string {
+    return this.#name;
+  }
+
+  public constructor(
+    client: RestfulLanceDBClient,
+    tableName: string,
+    dbName: string,
+  ) {
+    super();
+    this.#client = client;
+    this.#name = tableName;
+    this.#dbName = dbName;
+  }
+
+  isOpen(): boolean {
+    return !this.#client.isOpen();
+  }
+
+  close(): void {
+    this.#client.close();
+  }
+
+  display(): string {
+    return `RemoteTable(${this.#dbName}; ${this.#name})`;
+  }
+
+  async schema(): Promise<import("apache-arrow").Schema> {
+    const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+    // TODO: parse this into a valid arrow schema
+    return resp.schema;
+  }
+  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+    const { buf, mode } = await Table.parseTableData(
+      data,
+      options as CreateTableOptions,
+      true,
+    );
+    await this.#client.post(`${this.#tablePrefix}/insert/`, buf, {
+      params: {
+        mode,
+      },
+      headers: {
+        "Content-Type": "application/vnd.apache.arrow.stream",
+      },
+    });
+  }
+
+  async update(
+    optsOrUpdates:
+      | (Map<string, string> | Record<string, string>)
+      | ({
+          values: Map<string, IntoSql> | Record<string, IntoSql>;
+        } & Partial<UpdateOptions>)
+      | ({
+          valuesSql: Map<string, string> | Record<string, string>;
+        } & Partial<UpdateOptions>),
+    options?: Partial<UpdateOptions>,
+  ): Promise<void> {
+    const isValues =
+      "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
+    const isValuesSql =
+      "valuesSql" in optsOrUpdates &&
+      typeof optsOrUpdates.valuesSql !== "string";
+    const isMap = (obj: unknown): obj is Map<string, string> => {
+      return obj instanceof Map;
+    };
+
+    let predicate;
+    let columns: [string, string][];
+    switch (true) {
+      case isMap(optsOrUpdates):
+        columns = Array.from(optsOrUpdates.entries());
+        predicate = options?.where;
+        break;
+      case isValues && isMap(optsOrUpdates.values):
+        columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
+          k,
+          toSQL(v),
+        ]);
+        predicate = optsOrUpdates.where;
+        break;
+      case isValues && !isMap(optsOrUpdates.values):
+        columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
+          k,
+          toSQL(v),
+        ]);
+        predicate = optsOrUpdates.where;
+        break;
+
+      case isValuesSql && isMap(optsOrUpdates.valuesSql):
+        columns = Array.from(optsOrUpdates.valuesSql.entries());
+        predicate = optsOrUpdates.where;
+        break;
+      case isValuesSql && !isMap(optsOrUpdates.valuesSql):
+        columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
+          k,
+          v,
+        ]);
+        predicate = optsOrUpdates.where;
+        break;
+      default:
+        columns = Object.entries(optsOrUpdates as Record<string, string>);
+        predicate = options?.where;
+    }
+
+    await this.#client.post(`${this.#tablePrefix}/update/`, {
+      predicate: predicate ?? null,
+      updates: columns,
+    });
+  }
+  async countRows(filter?: unknown): Promise<number> {
+    const payload = { predicate: filter };
+    return await this.#client.post(`${this.#tablePrefix}/count_rows/`, payload);
+  }
+
+  async delete(predicate: unknown): Promise<void> {
+    const payload = { predicate };
+    await this.#client.post(`${this.#tablePrefix}/delete/`, payload);
+  }
+  async createIndex(
+    column: string,
+    options?: Partial<IndexOptions>,
+  ): Promise<void> {
+    if (options !== undefined) {
+      console.warn("options are not yet supported on the LanceDB cloud");
+    }
+    const indexType = "vector";
+    const metric = "L2";
+    const data = {
+      column,
+      // biome-ignore lint/style/useNamingConvention: external API
+      index_type: indexType,
+      // biome-ignore lint/style/useNamingConvention: external API
+      metric_type: metric,
+    };
+    await this.#client.post(`${this.#tablePrefix}/create_index`, data);
+  }
+  query(): import("..").Query {
+    throw new Error("query() is not yet supported on the LanceDB cloud");
+  }
+
+  search(_query: string | IntoVector): VectorQuery {
+    throw new Error("search() is not yet supported on the LanceDB cloud");
+  }
+  vectorSearch(_vector: unknown): import("..").VectorQuery {
+    throw new Error("vectorSearch() is not yet supported on the LanceDB cloud");
+  }
+  addColumns(_newColumnTransforms: unknown): Promise<void> {
+    throw new Error("addColumns() is not yet supported on the LanceDB cloud");
+  }
+  alterColumns(_columnAlterations: unknown): Promise<void> {
+    throw new Error("alterColumns() is not yet supported on the LanceDB cloud");
+  }
+  dropColumns(_columnNames: unknown): Promise<void> {
+    throw new Error("dropColumns() is not yet supported on the LanceDB cloud");
+  }
+  async version(): Promise<number> {
+    const resp = await this.#client.post(`${this.#tablePrefix}/describe/`);
+    return resp.version;
+  }
+  checkout(_version: unknown): Promise<void> {
+    throw new Error("checkout() is not yet supported on the LanceDB cloud");
+  }
+  checkoutLatest(): Promise<void> {
+    throw new Error(
+      "checkoutLatest() is not yet supported on the LanceDB cloud",
+    );
+  }
+  restore(): Promise<void> {
+    throw new Error("restore() is not yet supported on the LanceDB cloud");
+  }
+  optimize(_options?: unknown): Promise<import("../native").OptimizeStats> {
+    throw new Error("optimize() is not yet supported on the LanceDB cloud");
+  }
+  async listIndices(): Promise<import("../native").IndexConfig[]> {
+    return await this.#client.post(`${this.#tablePrefix}/index/list/`);
+  }
+  toArrow(): Promise<ArrowTable> {
+    throw new Error("toArrow() is not yet supported on the LanceDB cloud");
+  }
+  mergeInsert(_on: string | string[]): MergeInsertBuilder {
+    throw new Error("mergeInsert() is not yet supported on the LanceDB cloud");
+  }
+  async indexStats(_name: string): Promise<IndexStatistics | undefined> {
+    throw new Error("indexStats() is not yet supported on the LanceDB cloud");
+  }
+}

nodejs/lancedb/sanitize.ts

@@ -20,10 +20,12 @@
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
 
+import { BufferType, Data } from "apache-arrow";
 import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 import {
   Binary,
   Bool,
+  DataLike,
   DataType,
   DateDay,
   DateMillisecond,
@@ -56,9 +58,14 @@ import {
   Map_,
   Null,
   type Precision,
+  RecordBatch,
+  RecordBatchLike,
   Schema,
+  SchemaLike,
   SparseUnion,
   Struct,
+  Table,
+  TableLike,
   Time,
   TimeMicrosecond,
   TimeMillisecond,
@@ -488,7 +495,7 @@ export function sanitizeField(fieldLike: unknown): Field {
  * instance because they might be using a different instance of apache-arrow
  * than lancedb is using.
  */
-export function sanitizeSchema(schemaLike: unknown): Schema {
+export function sanitizeSchema(schemaLike: SchemaLike): Schema {
   if (schemaLike instanceof Schema) {
     return schemaLike;
   }
@@ -514,3 +521,68 @@ export function sanitizeSchema(schemaLike: unknown): Schema {
   );
   return new Schema(sanitizedFields, metadata);
 }
+
+export function sanitizeTable(tableLike: TableLike): Table {
+  if (tableLike instanceof Table) {
+    return tableLike;
+  }
+  if (typeof tableLike !== "object" || tableLike === null) {
+    throw Error("Expected a Table but object was null/undefined");
+  }
+  if (!("schema" in tableLike)) {
+    throw Error(
+      "The table passed in does not appear to be a table (no 'schema' property)",
+    );
+  }
+  if (!("batches" in tableLike)) {
+    throw Error(
+      "The table passed in does not appear to be a table (no 'columns' property)",
+    );
+  }
+  const schema = sanitizeSchema(tableLike.schema);
+
+  const batches = tableLike.batches.map(sanitizeRecordBatch);
+  return new Table(schema, batches);
+}
+
+function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
+  if (batchLike instanceof RecordBatch) {
+    return batchLike;
+  }
+  if (typeof batchLike !== "object" || batchLike === null) {
+    throw Error("Expected a RecordBatch but object was null/undefined");
+  }
+  if (!("schema" in batchLike)) {
+    throw Error(
+      "The record batch passed in does not appear to be a record batch (no 'schema' property)",
+    );
+  }
+  if (!("data" in batchLike)) {
+    throw Error(
+      "The record batch passed in does not appear to be a record batch (no 'data' property)",
+    );
+  }
+  const schema = sanitizeSchema(batchLike.schema);
+  const data = sanitizeData(batchLike.data);
+  return new RecordBatch(schema, data);
+}
+function sanitizeData(
+  dataLike: DataLike,
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+): import("apache-arrow").Data<Struct<any>> {
+  if (dataLike instanceof Data) {
+    return dataLike;
+  }
+  return new Data(
+    dataLike.type,
+    dataLike.offset,
+    dataLike.length,
+    dataLike.nullCount,
+    {
+      [BufferType.OFFSET]: dataLike.valueOffsets,
+      [BufferType.DATA]: dataLike.values,
+      [BufferType.VALIDITY]: dataLike.nullBitmap,
+      [BufferType.TYPE]: dataLike.typeIds,
+    },
+  );
+}

nodejs/lancedb/table.ts

@@ -17,20 +17,30 @@ import {
   Data,
   IntoVector,
   Schema,
+  TableLike,
   fromDataToBuffer,
+  fromTableToBuffer,
+  fromTableToStreamBuffer,
+  isArrowTable,
+  makeArrowTable,
   tableFromIPC,
 } from "./arrow";
+import { CreateTableOptions } from "./connection";
 
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
 import { IndexOptions } from "./indices";
+import { MergeInsertBuilder } from "./merge";
 import {
   AddColumnsSql,
   ColumnAlteration,
   IndexConfig,
+  IndexStatistics,
   OptimizeStats,
   Table as _NativeTable,
 } from "./native";
 import { Query, VectorQuery } from "./query";
+import { sanitizeTable } from "./sanitize";
+import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
 
 /**
@@ -88,19 +98,15 @@ export interface OptimizeOptions {
  * Closing a table is optional.  It not closed, it will be closed when it is garbage
  * collected.
  */
-export class Table {
+export abstract class Table {
-  private readonly inner: _NativeTable;
+  [Symbol.for("nodejs.util.inspect.custom")](): string {
-
+    return this.display();
-  /** Construct a Table. Internal use only. */
-  constructor(inner: _NativeTable) {
-    this.inner = inner;
   }
+  /** Returns the name of the table */
+  abstract get name(): string;
 
   /** Return true if the table has not been closed */
-  isOpen(): boolean {
+  abstract isOpen(): boolean;
-    return this.inner.isOpen();
-  }
-
   /**
    * Close the table, releasing any underlying resources.
    *
@@ -108,48 +114,44 @@ export class Table {
    *
    * Any attempt to use the table after it is closed will result in an error.
    */
-  close(): void {
+  abstract close(): void;
-    this.inner.close();
-  }
-
   /** Return a brief description of the table */
-  display(): string {
+  abstract display(): string;
-    return this.inner.display();
-  }
-
-  async #getEmbeddingFunctions(): Promise<
-    Map<string, EmbeddingFunctionConfig>
-  > {
-    const schema = await this.schema();
-    const registry = getRegistry();
-    return registry.parseFunctions(schema.metadata);
-  }
-
   /** Get the schema of the table. */
-  async schema(): Promise<Schema> {
+  abstract schema(): Promise<Schema>;
-    const schemaBuf = await this.inner.schema();
-    const tbl = tableFromIPC(schemaBuf);
-    return tbl.schema;
-  }
-
   /**
    * Insert records into this Table.
    * @param {Data} data Records to be inserted into the Table
    */
-  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+  abstract add(data: Data, options?: Partial<AddDataOptions>): Promise<void>;
-    const mode = options?.mode ?? "append";
+  /**
-    const schema = await this.schema();
+   * Update existing records in the Table
-    const registry = getRegistry();
+   * @param opts.values The values to update. The keys are the column names and the values
-    const functions = registry.parseFunctions(schema.metadata);
+   * are the values to set.
-
+   * @example
-    const buffer = await fromDataToBuffer(
+   * ```ts
-      data,
+   * table.update({where:"x = 2", values:{"vector": [10, 10]}})
-      functions.values().next().value,
+   * ```
-      schema,
+   */
-    );
+  abstract update(
-    await this.inner.add(buffer, mode);
+    opts: {
-  }
+      values: Map<string, IntoSql> | Record<string, IntoSql>;
-
+    } & Partial<UpdateOptions>,
+  ): Promise<void>;
+  /**
+   * Update existing records in the Table
+   * @param opts.valuesSql The values to update. The keys are the column names and the values
+   * are the values to set. The values are SQL expressions.
+   * @example
+   * ```ts
+   * table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
+   * ```
+   */
+  abstract update(
+    opts: {
+      valuesSql: Map<string, string> | Record<string, string>;
+    } & Partial<UpdateOptions>,
+  ): Promise<void>;
   /**
    * Update existing records in the Table
    *
@@ -175,30 +177,15 @@ export class Table {
    * @param {Partial<UpdateOptions>} options - additional options to control
    * the update behavior
    */
-  async update(
+  abstract update(
     updates: Map<string, string> | Record<string, string>,
     options?: Partial<UpdateOptions>,
-  ) {
+  ): Promise<void>;
-    const onlyIf = options?.where;
-    let columns: [string, string][];
-    if (updates instanceof Map) {
-      columns = Array.from(updates.entries());
-    } else {
-      columns = Object.entries(updates);
-    }
-    await this.inner.update(onlyIf, columns);
-  }
 
   /** Count the total number of rows in the dataset. */
-  async countRows(filter?: string): Promise<number> {
+  abstract countRows(filter?: string): Promise<number>;
-    return await this.inner.countRows(filter);
-  }
-
   /** Delete the rows that satisfy the predicate. */
-  async delete(predicate: string): Promise<void> {
+  abstract delete(predicate: string): Promise<void>;
-    await this.inner.delete(predicate);
-  }
-
   /**
    * Create an index to speed up queries.
    *
@@ -206,6 +193,9 @@ export class Table {
    * Indices on vector columns will speed up vector searches.
    * Indices on scalar columns will speed up filtering (in both
    * vector and non-vector searches)
+   *
+   * @note We currently don't support custom named indexes,
+   * The index name will always be `${column}_idx`
    * @example
    * // If the column has a vector (fixed size list) data type then
    * // an IvfPq vector index will be created.
@@ -225,13 +215,10 @@ export class Table {
    * // Or create a Scalar index
    * await table.createIndex("my_float_col");
    */
-  async createIndex(column: string, options?: Partial<IndexOptions>) {
+  abstract createIndex(
-    // Bit of a hack to get around the fact that TS has no package-scope.
+    column: string,
-    // biome-ignore lint/suspicious/noExplicitAny: skip
+    options?: Partial<IndexOptions>,
-    const nativeIndex = (options?.config as any)?.inner;
+  ): Promise<void>;
-    await this.inner.createIndex(nativeIndex, column, options?.replace);
-  }
-
   /**
    * Create a {@link Query} Builder.
    *
@@ -282,44 +269,20 @@ export class Table {
    * }
    * @returns {Query} A builder that can be used to parameterize the query
    */
-  query(): Query {
+  abstract query(): Query;
-    return new Query(this.inner);
-  }
-
   /**
    * Create a search query to find the nearest neighbors
    * of the given query vector
    * @param {string} query - the query. This will be converted to a vector using the table's provided embedding function
-   * @rejects {Error} If no embedding functions are defined in the table
+   * @note  If no embedding functions are defined in the table, this will error when collecting the results.
    */
-  search(query: string): Promise<VectorQuery>;
+  abstract search(query: string): VectorQuery;
   /**
    * Create a search query to find the nearest neighbors
    * of the given query vector
    * @param {IntoVector} query - the query vector
    */
-  search(query: IntoVector): VectorQuery;
+  abstract search(query: IntoVector): VectorQuery;
-  search(query: string | IntoVector): Promise<VectorQuery> | VectorQuery {
-    if (typeof query !== "string") {
-      return this.vectorSearch(query);
-    } else {
-      return this.#getEmbeddingFunctions().then(async (functions) => {
-        // TODO: Support multiple embedding functions
-        const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
-          .values()
-          .next().value;
-        if (!embeddingFunc) {
-          return Promise.reject(
-            new Error("No embedding functions are defined in the table"),
-          );
-        }
-        const embeddings =
-          await embeddingFunc.function.computeQueryEmbeddings(query);
-        return this.query().nearestTo(embeddings);
-      });
-    }
-  }
-
   /**
    * Search the table with a given query vector.
    *
@@ -327,11 +290,7 @@ export class Table {
    * is the same thing as calling `nearestTo` on the builder returned
    * by `query`.  @see {@link Query#nearestTo} for more details.
    */
-  vectorSearch(vector: IntoVector): VectorQuery {
+  abstract vectorSearch(vector: IntoVector): VectorQuery;
-    return this.query().nearestTo(vector);
-  }
-
-  // TODO: Support BatchUDF
   /**
    * Add new columns with defined values.
    * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
@@ -339,19 +298,14 @@ export class Table {
    * expressions will be evaluated for each row in the table, and can
    * reference existing columns in the table.
    */
-  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
+  abstract addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void>;
-    await this.inner.addColumns(newColumnTransforms);
-  }
 
   /**
    * Alter the name or nullability of columns.
    * @param {ColumnAlteration[]} columnAlterations One or more alterations to
    * apply to columns.
    */
-  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
+  abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
-    await this.inner.alterColumns(columnAlterations);
-  }
-
   /**
    * Drop one or more columns from the dataset
    *
@@ -363,15 +317,10 @@ export class Table {
    * be nested column references (e.g. "a.b.c") or top-level column names
    * (e.g. "a").
    */
-  async dropColumns(columnNames: string[]): Promise<void> {
+  abstract dropColumns(columnNames: string[]): Promise<void>;
-    await this.inner.dropColumns(columnNames);
-  }
-
   /** Retrieve the version of the table */
-  async version(): Promise<number> {
-    return await this.inner.version();
-  }
 
+  abstract version(): Promise<number>;
   /**
    * Checks out a specific version of the table _This is an in-place operation._
    *
@@ -397,19 +346,14 @@ export class Table {
    * console.log(await table.version()); // 2
    * ```
    */
-  async checkout(version: number): Promise<void> {
+  abstract checkout(version: number): Promise<void>;
-    await this.inner.checkout(version);
-  }
-
   /**
    * Checkout the latest version of the table. _This is an in-place operation._
    *
    * The table will be set back into standard mode, and will track the latest
    * version of the table.
    */
-  async checkoutLatest(): Promise<void> {
+  abstract checkoutLatest(): Promise<void>;
-    await this.inner.checkoutLatest();
-  }
 
   /**
    * Restore the table to the currently checked out version
@@ -423,10 +367,7 @@ export class Table {
    * Once the operation concludes the table will no longer be in a checked
    * out state and the read_consistency_interval, if any, will apply.
    */
-  async restore(): Promise<void> {
+  abstract restore(): Promise<void>;
-    await this.inner.restore();
-  }
-
   /**
    * Optimize the on-disk data and indices for better performance.
    *
@@ -457,6 +398,243 @@ export class Table {
    *  you have added or modified 100,000 or more records or run more than 20 data
    *  modification operations.
    */
+  abstract optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats>;
+  /** List all indices that have been created with {@link Table.createIndex} */
+  abstract listIndices(): Promise<IndexConfig[]>;
+  /** Return the table as an arrow table */
+  abstract toArrow(): Promise<ArrowTable>;
+
+  abstract mergeInsert(on: string | string[]): MergeInsertBuilder;
+
+  /** List all the stats of a specified index
+   *
+   * @param {string} name The name of the index.
+   * @returns {IndexStatistics | undefined} The stats of the index. If the index does not exist, it will return undefined
+   */
+  abstract indexStats(name: string): Promise<IndexStatistics | undefined>;
+
+  static async parseTableData(
+    data: Record<string, unknown>[] | TableLike,
+    options?: Partial<CreateTableOptions>,
+    streaming = false,
+  ) {
+    let mode: string = options?.mode ?? "create";
+    const existOk = options?.existOk ?? false;
+
+    if (mode === "create" && existOk) {
+      mode = "exist_ok";
+    }
+
+    let table: ArrowTable;
+    if (isArrowTable(data)) {
+      table = sanitizeTable(data);
+    } else {
+      table = makeArrowTable(data as Record<string, unknown>[], options);
+    }
+    if (streaming) {
+      const buf = await fromTableToStreamBuffer(
+        table,
+        options?.embeddingFunction,
+        options?.schema,
+      );
+      return { buf, mode };
+    } else {
+      const buf = await fromTableToBuffer(
+        table,
+        options?.embeddingFunction,
+        options?.schema,
+      );
+      return { buf, mode };
+    }
+  }
+}
+
+export class LocalTable extends Table {
+  private readonly inner: _NativeTable;
+
+  constructor(inner: _NativeTable) {
+    super();
+    this.inner = inner;
+  }
+  get name(): string {
+    return this.inner.name;
+  }
+  isOpen(): boolean {
+    return this.inner.isOpen();
+  }
+
+  close(): void {
+    this.inner.close();
+  }
+
+  display(): string {
+    return this.inner.display();
+  }
+
+  private async getEmbeddingFunctions(): Promise<
+    Map<string, EmbeddingFunctionConfig>
+  > {
+    const schema = await this.schema();
+    const registry = getRegistry();
+    return registry.parseFunctions(schema.metadata);
+  }
+
+  /** Get the schema of the table. */
+  async schema(): Promise<Schema> {
+    const schemaBuf = await this.inner.schema();
+    const tbl = tableFromIPC(schemaBuf);
+    return tbl.schema;
+  }
+
+  async add(data: Data, options?: Partial<AddDataOptions>): Promise<void> {
+    const mode = options?.mode ?? "append";
+    const schema = await this.schema();
+    const registry = getRegistry();
+    const functions = registry.parseFunctions(schema.metadata);
+
+    const buffer = await fromDataToBuffer(
+      data,
+      functions.values().next().value,
+      schema,
+    );
+    await this.inner.add(buffer, mode);
+  }
+
+  async update(
+    optsOrUpdates:
+      | (Map<string, string> | Record<string, string>)
+      | ({
+          values: Map<string, IntoSql> | Record<string, IntoSql>;
+        } & Partial<UpdateOptions>)
+      | ({
+          valuesSql: Map<string, string> | Record<string, string>;
+        } & Partial<UpdateOptions>),
+    options?: Partial<UpdateOptions>,
+  ) {
+    const isValues =
+      "values" in optsOrUpdates && typeof optsOrUpdates.values !== "string";
+    const isValuesSql =
+      "valuesSql" in optsOrUpdates &&
+      typeof optsOrUpdates.valuesSql !== "string";
+    const isMap = (obj: unknown): obj is Map<string, string> => {
+      return obj instanceof Map;
+    };
+
+    let predicate;
+    let columns: [string, string][];
+    switch (true) {
+      case isMap(optsOrUpdates):
+        columns = Array.from(optsOrUpdates.entries());
+        predicate = options?.where;
+        break;
+      case isValues && isMap(optsOrUpdates.values):
+        columns = Array.from(optsOrUpdates.values.entries()).map(([k, v]) => [
+          k,
+          toSQL(v),
+        ]);
+        predicate = optsOrUpdates.where;
+        break;
+      case isValues && !isMap(optsOrUpdates.values):
+        columns = Object.entries(optsOrUpdates.values).map(([k, v]) => [
+          k,
+          toSQL(v),
+        ]);
+        predicate = optsOrUpdates.where;
+        break;
+
+      case isValuesSql && isMap(optsOrUpdates.valuesSql):
+        columns = Array.from(optsOrUpdates.valuesSql.entries());
+        predicate = optsOrUpdates.where;
+        break;
+      case isValuesSql && !isMap(optsOrUpdates.valuesSql):
+        columns = Object.entries(optsOrUpdates.valuesSql).map(([k, v]) => [
+          k,
+          v,
+        ]);
+        predicate = optsOrUpdates.where;
+        break;
+      default:
+        columns = Object.entries(optsOrUpdates as Record<string, string>);
+        predicate = options?.where;
+    }
+    await this.inner.update(predicate, columns);
+  }
+
+  async countRows(filter?: string): Promise<number> {
+    return await this.inner.countRows(filter);
+  }
+
+  async delete(predicate: string): Promise<void> {
+    await this.inner.delete(predicate);
+  }
+
+  async createIndex(column: string, options?: Partial<IndexOptions>) {
+    // Bit of a hack to get around the fact that TS has no package-scope.
+    // biome-ignore lint/suspicious/noExplicitAny: skip
+    const nativeIndex = (options?.config as any)?.inner;
+    await this.inner.createIndex(nativeIndex, column, options?.replace);
+  }
+
+  query(): Query {
+    return new Query(this.inner);
+  }
+  search(query: string | IntoVector): VectorQuery {
+    if (typeof query !== "string") {
+      return this.vectorSearch(query);
+    } else {
+      const queryPromise = this.getEmbeddingFunctions().then(
+        async (functions) => {
+          // TODO: Support multiple embedding functions
+          const embeddingFunc: EmbeddingFunctionConfig | undefined = functions
+            .values()
+            .next().value;
+          if (!embeddingFunc) {
+            return Promise.reject(
+              new Error("No embedding functions are defined in the table"),
+            );
+          }
+          return await embeddingFunc.function.computeQueryEmbeddings(query);
+        },
+      );
+
+      return this.query().nearestTo(queryPromise);
+    }
+  }
+
+  vectorSearch(vector: IntoVector): VectorQuery {
+    return this.query().nearestTo(vector);
+  }
+
+  // TODO: Support BatchUDF
+
+  async addColumns(newColumnTransforms: AddColumnsSql[]): Promise<void> {
+    await this.inner.addColumns(newColumnTransforms);
+  }
+
+  async alterColumns(columnAlterations: ColumnAlteration[]): Promise<void> {
+    await this.inner.alterColumns(columnAlterations);
+  }
+
+  async dropColumns(columnNames: string[]): Promise<void> {
+    await this.inner.dropColumns(columnNames);
+  }
+
+  async version(): Promise<number> {
+    return await this.inner.version();
+  }
+
+  async checkout(version: number): Promise<void> {
+    await this.inner.checkout(version);
+  }
+
+  async checkoutLatest(): Promise<void> {
+    await this.inner.checkoutLatest();
+  }
+
+  async restore(): Promise<void> {
+    await this.inner.restore();
+  }
+
   async optimize(options?: Partial<OptimizeOptions>): Promise<OptimizeStats> {
     let cleanupOlderThanMs;
     if (
@@ -469,13 +647,23 @@ export class Table {
     return await this.inner.optimize(cleanupOlderThanMs);
   }
 
-  /** List all indices that have been created with {@link Table.createIndex} */
   async listIndices(): Promise<IndexConfig[]> {
     return await this.inner.listIndices();
   }
 
-  /** Return the table as an arrow table */
   async toArrow(): Promise<ArrowTable> {
     return await this.query().toArrow();
   }
+
+  async indexStats(name: string): Promise<IndexStatistics | undefined> {
+    const stats = await this.inner.indexStats(name);
+    if (stats === null) {
+      return undefined;
+    }
+    return stats;
+  }
+  mergeInsert(on: string | string[]): MergeInsertBuilder {
+    on = Array.isArray(on) ? on : [on];
+    return new MergeInsertBuilder(this.inner.mergeInsert(on));
+  }
 }

nodejs/lancedb/util.ts

@@ -0,0 +1,69 @@
+export type IntoSql =
+  | string
+  | number
+  | boolean
+  | null
+  | Date
+  | ArrayBufferLike
+  | Buffer
+  | IntoSql[];
+
+export function toSQL(value: IntoSql): string {
+  if (typeof value === "string") {
+    return `'${value.replace(/'/g, "''")}'`;
+  } else if (typeof value === "number") {
+    return value.toString();
+  } else if (typeof value === "boolean") {
+    return value ? "TRUE" : "FALSE";
+  } else if (value === null) {
+    return "NULL";
+  } else if (value instanceof Date) {
+    return `'${value.toISOString()}'`;
+  } else if (Array.isArray(value)) {
+    return `[${value.map(toSQL).join(", ")}]`;
+  } else if (Buffer.isBuffer(value)) {
+    return `X'${value.toString("hex")}'`;
+  } else if (value instanceof ArrayBuffer) {
+    return `X'${Buffer.from(value).toString("hex")}'`;
+  } else {
+    throw new Error(
+      `Unsupported value type: ${typeof value} value: (${value})`,
+    );
+  }
+}
+
+export class TTLCache {
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  private readonly cache: Map<string, { value: any; expires: number }>;
+
+  /**
+   * @param ttl Time to live in milliseconds
+   */
+  constructor(private readonly ttl: number) {
+    this.cache = new Map();
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  get(key: string): any | undefined {
+    const entry = this.cache.get(key);
+    if (entry === undefined) {
+      return undefined;
+    }
+
+    if (entry.expires < Date.now()) {
+      this.cache.delete(key);
+      return undefined;
+    }
+
+    return entry.value;
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  set(key: string, value: any): void {
+    this.cache.set(key, { value, expires: Date.now() + this.ttl });
+  }
+
+  delete(key: string): void {
+    this.cache.delete(key);
+  }
+}

nodejs/native.d.ts

@@ -0,0 +1,208 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/* auto-generated by NAPI-RS */
+
+/** A description of an index currently configured on a column */
+export interface IndexConfig {
+  /** The name of the index */
+  name: string
+  /** The type of the index */
+  indexType: string
+  /**
+   * The columns in the index
+   *
+   * Currently this is always an array of size 1. In the future there may
+   * be more columns to represent composite indices.
+   */
+  columns: Array<string>
+}
+/** Statistics about a compaction operation. */
+export interface CompactionStats {
+  /** The number of fragments removed */
+  fragmentsRemoved: number
+  /** The number of new, compacted fragments added */
+  fragmentsAdded: number
+  /** The number of data files removed */
+  filesRemoved: number
+  /** The number of new, compacted data files added */
+  filesAdded: number
+}
+/** Statistics about a cleanup operation */
+export interface RemovalStats {
+  /** The number of bytes removed */
+  bytesRemoved: number
+  /** The number of old versions removed */
+  oldVersionsRemoved: number
+}
+/** Statistics about an optimize operation */
+export interface OptimizeStats {
+  /** Statistics about the compaction operation */
+  compaction: CompactionStats
+  /** Statistics about the removal operation */
+  prune: RemovalStats
+}
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  /**
+   * The new name of the column. If not provided then the name will not be changed.
+   * This must be distinct from the names of all other columns in the table.
+   */
+  rename?: string
+  /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+  nullable?: boolean
+}
+/** A definition of a new column to add to a table. */
+export interface AddColumnsSql {
+  /** The name of the new column. */
+  name: string
+  /**
+   * The values to populate the new column with, as a SQL expression.
+   * The expression can reference other columns in the table.
+   */
+  valueSql: string
+}
+export interface IndexStatistics {
+  /** The number of rows indexed by the index */
+  numIndexedRows: number
+  /** The number of rows not indexed */
+  numUnindexedRows: number
+  /** The type of the index */
+  indexType?: string
+  /** The metadata for each index */
+  indices: Array<IndexMetadata>
+}
+export interface IndexMetadata {
+  metricType?: string
+  indexType?: string
+}
+export interface ConnectionOptions {
+  /**
+   * (For LanceDB OSS only): The interval, in seconds, at which to check for
+   * updates to the table from other processes. If None, then consistency is not
+   * checked. For performance reasons, this is the default. For strong
+   * consistency, set this to zero seconds. Then every read will check for
+   * updates from other processes. As a compromise, you can set this to a
+   * non-zero value for eventual consistency. If more than that interval
+   * has passed since the last check, then the table will be checked for updates.
+   * Note: this consistency only applies to read operations. Write operations are
+   * always consistent.
+   */
+  readConsistencyInterval?: number
+  /**
+   * (For LanceDB OSS only): configuration for object storage.
+   *
+   * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+   */
+  storageOptions?: Record<string, string>
+}
+/** Write mode for writing a table. */
+export const enum WriteMode {
+  Create = 'Create',
+  Append = 'Append',
+  Overwrite = 'Overwrite'
+}
+/** Write options when creating a Table. */
+export interface WriteOptions {
+  /** Write mode for writing to a table. */
+  mode?: WriteMode
+}
+export interface OpenTableOptions {
+  storageOptions?: Record<string, string>
+}
+export class Connection {
+  /** Create a new Connection instance from the given URI. */
+  static new(uri: string, options: ConnectionOptions): Promise<Connection>
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** List all tables in the dataset. */
+  tableNames(startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
+  /**
+   * Create table from a Apache Arrow IPC (file) buffer.
+   *
+   * Parameters:
+   * - name: The name of the table.
+   * - buf: The buffer containing the IPC file.
+   *
+   */
+  createTable(name: string, buf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
+  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, storageOptions?: Record<string, string> | undefined | null, useLegacyFormat?: boolean | undefined | null): Promise<Table>
+  openTable(name: string, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
+  /** Drop table with the name. Or raise an error if the table does not exist. */
+  dropTable(name: string): Promise<void>
+}
+export class Index {
+  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
+  static btree(): Index
+}
+/** Typescript-style Async Iterator over RecordBatches  */
+export class RecordBatchIterator {
+  next(): Promise<Buffer | null>
+}
+/** A builder used to create and run a merge insert operation */
+export class NativeMergeInsertBuilder {
+  whenMatchedUpdateAll(condition?: string | undefined | null): NativeMergeInsertBuilder
+  whenNotMatchedInsertAll(): NativeMergeInsertBuilder
+  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
+  execute(buf: Buffer): Promise<void>
+}
+export class Query {
+  onlyIf(predicate: string): void
+  select(columns: Array<[string, string]>): void
+  limit(limit: number): void
+  nearestTo(vector: Float32Array): VectorQuery
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
+}
+export class VectorQuery {
+  column(column: string): void
+  distanceType(distanceType: string): void
+  postfilter(): void
+  refineFactor(refineFactor: number): void
+  nprobes(nprobe: number): void
+  bypassVectorIndex(): void
+  onlyIf(predicate: string): void
+  select(columns: Array<[string, string]>): void
+  limit(limit: number): void
+  execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
+  explainPlan(verbose: boolean): Promise<string>
+}
+export class Table {
+  name: string
+  display(): string
+  isOpen(): boolean
+  close(): void
+  /** Return Schema as empty Arrow IPC file. */
+  schema(): Promise<Buffer>
+  add(buf: Buffer, mode: string): Promise<void>
+  countRows(filter?: string | undefined | null): Promise<number>
+  delete(predicate: string): Promise<void>
+  createIndex(index: Index | undefined | null, column: string, replace?: boolean | undefined | null): Promise<void>
+  update(onlyIf: string | undefined | null, columns: Array<[string, string]>): Promise<void>
+  query(): Query
+  vectorSearch(vector: Float32Array): VectorQuery
+  addColumns(transforms: Array<AddColumnsSql>): Promise<void>
+  alterColumns(alterations: Array<ColumnAlteration>): Promise<void>
+  dropColumns(columns: Array<string>): Promise<void>
+  version(): Promise<number>
+  checkout(version: number): Promise<void>
+  checkoutLatest(): Promise<void>
+  restore(): Promise<void>
+  optimize(olderThanMs?: number | undefined | null): Promise<OptimizeStats>
+  listIndices(): Promise<Array<IndexConfig>>
+  indexStats(indexName: string): Promise<IndexStatistics | null>
+  mergeInsert(on: Array<string>): NativeMergeInsertBuilder
+}

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.5.1",
+	"version": "0.7.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.5.1",
+	"version": "0.7.0",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.5.1",
+	"version": "0.7.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.5.1",
+	"version": "0.7.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.5.1",
+	"version": "0.7.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -1,6 +1,16 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.5.1",
+  "description": "LanceDB: A serverless, low-latency vector database for AI applications",
+  "keywords": [
+    "database",
+    "lance",
+    "lancedb",
+    "search",
+    "vector",
+    "vector database",
+    "ann"
+  ],
+  "version": "0.7.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -24,9 +34,10 @@
   "devDependencies": {
     "@aws-sdk/client-kms": "^3.33.0",
     "@aws-sdk/client-s3": "^3.33.0",
+    "@aws-sdk/client-dynamodb": "^3.33.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
-    "@napi-rs/cli": "^2.18.0",
+    "@napi-rs/cli": "^2.18.3",
     "@types/jest": "^29.1.2",
     "@types/tmp": "^0.2.6",
     "apache-arrow-old": "npm:apache-arrow@13.0.0",
@@ -38,7 +49,8 @@
     "typedoc": "^0.25.7",
     "typedoc-plugin-markdown": "^3.17.1",
     "typescript": "^5.3.3",
-    "typescript-eslint": "^7.1.0"
+    "typescript-eslint": "^7.1.0",
+    "@types/axios": "^0.14.0"
   },
   "ava": {
     "timeout": "3m"
@@ -57,7 +69,7 @@
     "lint-ci": "biome ci .",
     "docs": "typedoc --plugin typedoc-plugin-markdown --out ../docs/src/js lancedb/index.ts",
     "lint": "biome check . && biome format .",
-    "lint-fix": "biome check --apply-unsafe  . && biome format --write .",
+    "lint-fix": "biome check --write . && biome format --write .",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
     "integration": "S3_TEST=1 npm run test",
@@ -65,8 +77,13 @@
     "version": "napi version"
   },
   "dependencies": {
-    "apache-arrow": "^15.0.0",
+    "axios": "^1.7.2",
-    "openai": "^4.29.2",
     "reflect-metadata": "^0.2.2"
+  },
+  "optionalDependencies": {
+    "openai": "^4.29.2"
+  },
+  "peerDependencies": {
+    "apache-arrow": "^15.0.0"
   }
 }

nodejs/src/connection.rs

@@ -56,12 +56,6 @@ impl Connection {
     #[napi(factory)]
     pub async fn new(uri: String, options: ConnectionOptions) -> napi::Result<Self> {
         let mut builder = ConnectBuilder::new(&uri);
-        if let Some(api_key) = options.api_key {
-            builder = builder.api_key(&api_key);
-        }
-        if let Some(host_override) = options.host_override {
-            builder = builder.host_override(&host_override);
-        }
         if let Some(interval) = options.read_consistency_interval {
             builder =
                 builder.read_consistency_interval(std::time::Duration::from_secs_f64(interval));
@@ -95,7 +89,7 @@ impl Connection {
     }
 
     /// List all tables in the dataset.
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn table_names(
         &self,
         start_after: Option<String>,
@@ -119,7 +113,7 @@ impl Connection {
     /// - name: The name of the table.
     /// - buf: The buffer containing the IPC file.
     ///
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn create_table(
         &self,
         name: String,
@@ -147,7 +141,7 @@ impl Connection {
         Ok(Table::new(tbl))
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn create_empty_table(
         &self,
         name: String,
@@ -179,7 +173,7 @@ impl Connection {
         Ok(Table::new(tbl))
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn open_table(
         &self,
         name: String,
@@ -203,7 +197,7 @@ impl Connection {
     }
 
     /// Drop table with the name. Or raise an error if the table does not exist.
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn drop_table(&self, name: String) -> napi::Result<()> {
         self.get_inner()?
             .drop_table(&name)

nodejs/src/iterator.rs

@@ -30,7 +30,7 @@ impl RecordBatchIterator {
         Self { inner }
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async unsafe fn next(&mut self) -> napi::Result<Option<Buffer>> {
         if let Some(rst) = self.inner.next().await {
             let batch = rst.map_err(|e| {

nodejs/src/lib.rs

@@ -20,6 +20,7 @@ mod connection;
 mod error;
 mod index;
 mod iterator;
+pub mod merge;
 mod query;
 mod table;
 mod util;
@@ -27,8 +28,6 @@ mod util;
 #[napi(object)]
 #[derive(Debug)]
 pub struct ConnectionOptions {
-    pub api_key: Option<String>,
-    pub host_override: Option<String>,
     /// (For LanceDB OSS only): The interval, in seconds, at which to check for
     /// updates to the table from other processes. If None, then consistency is not
     /// checked. For performance reasons, this is the default. For strong

nodejs/src/merge.rs

@@ -0,0 +1,53 @@
+use lancedb::{arrow::IntoArrow, ipc::ipc_file_to_batches, table::merge::MergeInsertBuilder};
+use napi::bindgen_prelude::*;
+use napi_derive::napi;
+
+#[napi]
+#[derive(Clone)]
+/// A builder used to create and run a merge insert operation
+pub struct NativeMergeInsertBuilder {
+    pub(crate) inner: MergeInsertBuilder,
+}
+
+#[napi]
+impl NativeMergeInsertBuilder {
+    #[napi]
+    pub fn when_matched_update_all(&self, condition: Option<String>) -> Self {
+        let mut this = self.clone();
+        this.inner.when_matched_update_all(condition);
+        this
+    }
+
+    #[napi]
+    pub fn when_not_matched_insert_all(&self) -> Self {
+        let mut this = self.clone();
+        this.inner.when_not_matched_insert_all();
+        this
+    }
+    #[napi]
+    pub fn when_not_matched_by_source_delete(&self, filter: Option<String>) -> Self {
+        let mut this = self.clone();
+        this.inner.when_not_matched_by_source_delete(filter);
+        this
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn execute(&self, buf: Buffer) -> napi::Result<()> {
+        let data = ipc_file_to_batches(buf.to_vec())
+            .and_then(IntoArrow::into_arrow)
+            .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
+
+        let this = self.clone();
+
+        this.inner
+            .execute(data)
+            .await
+            .map_err(|e| napi::Error::from_reason(format!("Failed to execute merge insert: {}", e)))
+    }
+}
+
+impl From<MergeInsertBuilder> for NativeMergeInsertBuilder {
+    fn from(inner: MergeInsertBuilder) -> Self {
+        Self { inner }
+    }
+}

nodejs/src/query.rs

@@ -62,7 +62,7 @@ impl Query {
         Ok(VectorQuery { inner })
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn execute(
         &self,
         max_batch_length: Option<u32>,
@@ -80,6 +80,13 @@ impl Query {
             })?;
         Ok(RecordBatchIterator::new(inner_stream))
     }
+
+    #[napi]
+    pub async fn explain_plan(&self, verbose: bool) -> napi::Result<String> {
+        self.inner.explain_plan(verbose).await.map_err(|e| {
+            napi::Error::from_reason(format!("Failed to retrieve the query plan: {}", e))
+        })
+    }
 }
 
 #[napi]
@@ -136,7 +143,7 @@ impl VectorQuery {
         self.inner = self.inner.clone().limit(limit as usize);
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn execute(
         &self,
         max_batch_length: Option<u32>,
@@ -154,4 +161,11 @@ impl VectorQuery {
             })?;
         Ok(RecordBatchIterator::new(inner_stream))
     }
+
+    #[napi]
+    pub async fn explain_plan(&self, verbose: bool) -> napi::Result<String> {
+        self.inner.explain_plan(verbose).await.map_err(|e| {
+            napi::Error::from_reason(format!("Failed to retrieve the query plan: {}", e))
+        })
+    }
 }

nodejs/src/table.rs

@@ -23,13 +23,14 @@ use napi_derive::napi;
 
 use crate::error::NapiErrorExt;
 use crate::index::Index;
+use crate::merge::NativeMergeInsertBuilder;
 use crate::query::{Query, VectorQuery};
 
 #[napi]
 pub struct Table {
     // We keep a duplicate of the table name so we can use it for error
     // messages even if the table has been closed
-    name: String,
+    pub name: String,
     pub(crate) inner: Option<LanceDbTable>,
 }
 
@@ -69,7 +70,7 @@ impl Table {
     }
 
     /// Return Schema as empty Arrow IPC file.
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn schema(&self) -> napi::Result<Buffer> {
         let schema =
             self.inner_ref()?.schema().await.map_err(|e| {
@@ -85,7 +86,7 @@ impl Table {
         })?))
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn add(&self, buf: Buffer, mode: String) -> napi::Result<()> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
@@ -107,7 +108,7 @@ impl Table {
         })
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn count_rows(&self, filter: Option<String>) -> napi::Result<i64> {
         self.inner_ref()?
             .count_rows(filter)
@@ -121,7 +122,7 @@ impl Table {
             })
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn delete(&self, predicate: String) -> napi::Result<()> {
         self.inner_ref()?.delete(&predicate).await.map_err(|e| {
             napi::Error::from_reason(format!(
@@ -131,7 +132,7 @@ impl Table {
         })
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn create_index(
         &self,
         index: Option<&Index>,
@@ -150,7 +151,7 @@ impl Table {
         builder.execute().await.default_error()
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn update(
         &self,
         only_if: Option<String>,
@@ -166,17 +167,17 @@ impl Table {
         op.execute().await.default_error()
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub fn query(&self) -> napi::Result<Query> {
         Ok(Query::new(self.inner_ref()?.query()))
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub fn vector_search(&self, vector: Float32Array) -> napi::Result<VectorQuery> {
         self.query()?.nearest_to(vector)
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn add_columns(&self, transforms: Vec<AddColumnsSql>) -> napi::Result<()> {
         let transforms = transforms
             .into_iter()
@@ -195,7 +196,7 @@ impl Table {
         Ok(())
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn alter_columns(&self, alterations: Vec<ColumnAlteration>) -> napi::Result<()> {
         for alteration in &alterations {
             if alteration.rename.is_none() && alteration.nullable.is_none() {
@@ -221,7 +222,7 @@ impl Table {
         Ok(())
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn drop_columns(&self, columns: Vec<String>) -> napi::Result<()> {
         let col_refs = columns.iter().map(String::as_str).collect::<Vec<_>>();
         self.inner_ref()?
@@ -236,7 +237,7 @@ impl Table {
         Ok(())
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn version(&self) -> napi::Result<i64> {
         self.inner_ref()?
             .version()
@@ -245,7 +246,7 @@ impl Table {
             .default_error()
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn checkout(&self, version: i64) -> napi::Result<()> {
         self.inner_ref()?
             .checkout(version as u64)
@@ -253,17 +254,17 @@ impl Table {
             .default_error()
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn checkout_latest(&self) -> napi::Result<()> {
         self.inner_ref()?.checkout_latest().await.default_error()
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn restore(&self) -> napi::Result<()> {
         self.inner_ref()?.restore().await.default_error()
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn optimize(&self, older_than_ms: Option<i64>) -> napi::Result<OptimizeStats> {
         let inner = self.inner_ref()?;
 
@@ -317,7 +318,7 @@ impl Table {
         })
     }
 
-    #[napi]
+    #[napi(catch_unwind)]
     pub async fn list_indices(&self) -> napi::Result<Vec<IndexConfig>> {
         Ok(self
             .inner_ref()?
@@ -328,16 +329,31 @@ impl Table {
             .map(IndexConfig::from)
             .collect::<Vec<_>>())
     }
+
+    #[napi(catch_unwind)]
+    pub async fn index_stats(&self, index_name: String) -> napi::Result<Option<IndexStatistics>> {
+        let tbl = self.inner_ref()?.as_native().unwrap();
+        let stats = tbl.index_stats(&index_name).await.default_error()?;
+        Ok(stats.map(IndexStatistics::from))
+    }
+
+    #[napi(catch_unwind)]
+    pub fn merge_insert(&self, on: Vec<String>) -> napi::Result<NativeMergeInsertBuilder> {
+        let on: Vec<_> = on.iter().map(String::as_str).collect();
+        Ok(self.inner_ref()?.merge_insert(on.as_slice()).into())
+    }
 }
 
 #[napi(object)]
 /// A description of an index currently configured on a column
 pub struct IndexConfig {
+    /// The name of the index
+    pub name: String,
     /// The type of the index
     pub index_type: String,
     /// The columns in the index
     ///
-    /// Currently this is always an array of size 1.  In the future there may
+    /// Currently this is always an array of size 1. In the future there may
     /// be more columns to represent composite indices.
     pub columns: Vec<String>,
 }
@@ -348,6 +364,7 @@ impl From<lancedb::index::IndexConfig> for IndexConfig {
         Self {
             index_type,
             columns: value.columns,
+            name: value.name,
         }
     }
 }
@@ -430,3 +447,40 @@ pub struct AddColumnsSql {
     /// The expression can reference other columns in the table.
     pub value_sql: String,
 }
+
+#[napi(object)]
+pub struct IndexStatistics {
+    /// The number of rows indexed by the index
+    pub num_indexed_rows: f64,
+    /// The number of rows not indexed
+    pub num_unindexed_rows: f64,
+    /// The type of the index
+    pub index_type: Option<String>,
+    /// The metadata for each index
+    pub indices: Vec<IndexMetadata>,
+}
+impl From<lancedb::index::IndexStatistics> for IndexStatistics {
+    fn from(value: lancedb::index::IndexStatistics) -> Self {
+        Self {
+            num_indexed_rows: value.num_indexed_rows as f64,
+            num_unindexed_rows: value.num_unindexed_rows as f64,
+            index_type: value.index_type.map(|t| format!("{:?}", t)),
+            indices: value.indices.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+#[napi(object)]
+pub struct IndexMetadata {
+    pub metric_type: Option<String>,
+    pub index_type: Option<String>,
+}
+
+impl From<lancedb::index::IndexMetadata> for IndexMetadata {
+    fn from(value: lancedb::index::IndexMetadata) -> Self {
+        Self {
+            metric_type: value.metric_type,
+            index_type: value.index_type,
+        }
+    }
+}

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.8.2"
+current_version = "0.10.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.8.2"
+version = "0.10.1"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true

python/pyproject.toml

@@ -3,7 +3,7 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
     "deprecation",
-    "pylance==0.12.1",
+    "pylance==0.14.1",
     "ratelimiter~=1.0",
     "requests>=2.31.0",
     "retry>=0.9.2",
@@ -57,15 +57,10 @@ tests = [
     "duckdb",
     "pytz",
     "polars>=0.19",
-    "tantivy"
+    "tantivy",
 ]
 dev = ["ruff", "pre-commit"]
-docs = [
+docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
-    "mkdocs",
-    "mkdocs-jupyter",
-    "mkdocs-material",
-    "mkdocstrings[python]",
-]
 clip = ["torch", "pillow", "open-clip"]
 embeddings = [
     "openai>=1.6.1",
@@ -100,5 +95,5 @@ addopts = "--strict-markers --ignore-glob=lancedb/embeddings/*.py"
 markers = [
     "slow: marks tests as slow (deselect with '-m \"not slow\"')",
     "asyncio",
-    "s3_test"
+    "s3_test",
 ]

python/python/lancedb/__init__.py

@@ -15,7 +15,7 @@ import importlib.metadata
 import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Union, Any
 
 __version__ = importlib.metadata.version("lancedb")
 
@@ -35,7 +35,7 @@ def connect(
     host_override: Optional[str] = None,
     read_consistency_interval: Optional[timedelta] = None,
     request_thread_pool: Optional[Union[int, ThreadPoolExecutor]] = None,
-    **kwargs,
+    **kwargs: Any,
 ) -> DBConnection:
     """Connect to a LanceDB database.
 

python/python/lancedb/db.py

@@ -28,12 +28,11 @@ from lancedb.common import data_to_reader, validate_schema
 
 from ._lancedb import connect as lancedb_connect
 from .pydantic import LanceModel
-from .table import AsyncTable, LanceTable, Table, _sanitize_data
+from .table import AsyncTable, LanceTable, Table, _sanitize_data, _table_path
 from .util import (
     fs_from_uri,
     get_uri_location,
     get_uri_scheme,
-    join_uri,
     validate_table_name,
 )
 
@@ -457,16 +456,18 @@ class LanceDBConnection(DBConnection):
             If True, ignore if the table does not exist.
         """
         try:
-            filesystem, path = fs_from_uri(self.uri)
+            table_uri = _table_path(self.uri, name)
-            table_path = join_uri(path, name + ".lance")
+            filesystem, path = fs_from_uri(table_uri)
-            filesystem.delete_dir(table_path)
+            filesystem.delete_dir(path)
         except FileNotFoundError:
             if not ignore_missing:
                 raise
 
     @override
     def drop_database(self):
-        filesystem, path = fs_from_uri(self.uri)
+        dummy_table_uri = _table_path(self.uri, "dummy")
+        uri = dummy_table_uri.removesuffix("dummy.lance")
+        filesystem, path = fs_from_uri(uri)
         filesystem.delete_dir(path)
 
 

python/python/lancedb/embeddings/__init__.py

@@ -25,3 +25,4 @@ from .gte import GteEmbeddings
 from .transformers import TransformersEmbeddingFunction, ColbertEmbeddings
 from .imagebind import ImageBindEmbeddings
 from .utils import with_embeddings
+from .jinaai import JinaEmbeddings

python/python/lancedb/embeddings/jinaai.py

@@ -0,0 +1,236 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import io
+import requests
+import base64
+from urllib.parse import urlparse
+from pathlib import Path
+from typing import TYPE_CHECKING, ClassVar, List, Union, Optional, Any, Dict
+
+import numpy as np
+import pyarrow as pa
+
+from ..util import attempt_import_or_raise
+from .base import EmbeddingFunction
+from .registry import register
+from .utils import api_key_not_found_help, TEXT, IMAGES, url_retrieve
+
+if TYPE_CHECKING:
+    import PIL
+
+API_URL = "https://api.jina.ai/v1/embeddings"
+
+
+def is_valid_url(text):
+    try:
+        parsed = urlparse(text)
+        return bool(parsed.scheme) and bool(parsed.netloc)
+    except Exception:
+        return False
+
+
+@register("jina")
+class JinaEmbeddings(EmbeddingFunction):
+    """
+    An embedding function that uses the Jina API
+
+    https://jina.ai/embeddings/
+
+    Parameters
+    ----------
+    name: str, default "jina-clip-v1". Note that some models support both image
+        and text embeddings and some just text embedding
+
+    api_key: str, default None
+        The api key to access Jina API. If you pass None, you can set JINA_API_KEY
+        environment variable
+
+    """
+
+    name: str = "jina-clip-v1"
+    api_key: Optional[str] = None
+    _session: ClassVar = None
+
+    def ndims(self):
+        # TODO: fix hardcoding
+        return 768
+
+    def sanitize_input(
+        self, inputs: Union[TEXT, IMAGES]
+    ) -> Union[List[Any], np.ndarray]:
+        """
+        Sanitize the input to the embedding function.
+        """
+        if isinstance(inputs, (str, bytes, Path)):
+            inputs = [inputs]
+        elif isinstance(inputs, pa.Array):
+            inputs = inputs.to_pylist()
+        elif isinstance(inputs, pa.ChunkedArray):
+            inputs = inputs.combine_chunks().to_pylist()
+        else:
+            if isinstance(inputs, list):
+                inputs = inputs
+            else:
+                PIL = attempt_import_or_raise("PIL", "pillow")
+                if isinstance(inputs, PIL.Image.Image):
+                    inputs = [inputs]
+        return inputs
+
+    @staticmethod
+    def _generate_image_input_dict(image: Union[str, bytes, "PIL.Image.Image"]) -> Dict:
+        if isinstance(image, bytes):
+            image_dict = {"image": base64.b64encode(image).decode("utf-8")}
+        elif isinstance(image, (str, Path)):
+            parsed = urlparse.urlparse(image)
+            # TODO handle drive letter on windows.
+            PIL = attempt_import_or_raise("PIL", "pillow")
+            if parsed.scheme == "file":
+                pil_image = PIL.Image.open(parsed.path)
+            elif parsed.scheme == "":
+                pil_image = PIL.Image.open(image if os.name == "nt" else parsed.path)
+            elif parsed.scheme.startswith("http"):
+                pil_image = PIL.Image.open(io.BytesIO(url_retrieve(image)))
+            else:
+                raise NotImplementedError("Only local and http(s) urls are supported")
+            buffered = io.BytesIO()
+            pil_image.save(buffered, format="PNG")
+            image_bytes = buffered.getvalue()
+            image_dict = {"image": base64.b64encode(image_bytes).decode("utf-8")}
+        else:
+            PIL = attempt_import_or_raise("PIL", "pillow")
+
+            if isinstance(image, PIL.Image.Image):
+                buffered = io.BytesIO()
+                image.save(buffered, format="PNG")
+                image_bytes = buffered.getvalue()
+                image_dict = {"image": base64.b64encode(image_bytes).decode("utf-8")}
+            else:
+                raise TypeError(
+                    f"JinaEmbeddingFunction supports str, Path, bytes or PIL Image"
+                    f" as query, but {type(image)} is given"
+                )
+        return image_dict
+
+    def compute_query_embeddings(
+        self, query: Union[str, bytes, "Path", "PIL.Image.Image"], *args, **kwargs
+    ) -> List[np.ndarray]:
+        """
+        Compute the embeddings for a given user query
+
+        Parameters
+        ----------
+        query : Union[str, PIL.Image.Image]
+            The query to embed. A query can be either text or an image.
+        """
+        if isinstance(query, str):
+            if not is_valid_url(query):
+                return self.generate_text_embeddings([query])
+            else:
+                return [self.generate_image_embedding(query)]
+        elif isinstance(query, (Path, bytes)):
+            return [self.generate_image_embedding(query)]
+        else:
+            PIL = attempt_import_or_raise("PIL", "pillow")
+
+            if isinstance(query, PIL.Image.Image):
+                return [self.generate_image_embedding(query)]
+            else:
+                raise TypeError(
+                    f"JinaEmbeddingFunction supports str, Path, bytes or PIL Image"
+                    f" as query, but {type(query)} is given"
+                )
+
+    def compute_source_embeddings(
+        self, inputs: Union[TEXT, IMAGES], *args, **kwargs
+    ) -> List[np.array]:
+        inputs = self.sanitize_input(inputs)
+        model_inputs = []
+        image_inputs = 0
+
+        def process_input(input, model_inputs, image_inputs):
+            if isinstance(input, str):
+                if not is_valid_url(input):
+                    model_inputs.append({"text": input})
+                else:
+                    image_inputs += 1
+                    model_inputs.append(self._generate_image_input_dict(input))
+            elif isinstance(input, list):
+                for _input in input:
+                    image_inputs = process_input(_input, model_inputs, image_inputs)
+            else:
+                image_inputs += 1
+                model_inputs.append(self._generate_image_input_dict(input))
+            return image_inputs
+
+        for input in inputs:
+            image_inputs = process_input(input, model_inputs, image_inputs)
+
+        if image_inputs > 0:
+            return self._generate_embeddings(model_inputs)
+        else:
+            return self.generate_text_embeddings(inputs)
+
+    def generate_image_embedding(
+        self, image: Union[str, bytes, Path, "PIL.Image.Image"]
+    ) -> np.ndarray:
+        """
+        Generate the embedding for a single image
+
+        Parameters
+        ----------
+        image : Union[str, bytes, PIL.Image.Image]
+            The image to embed. If the image is a str, it is treated as a uri.
+            If the image is bytes, it is treated as the raw image bytes.
+        """
+        image_dict = self._generate_image_input_dict(image)
+        return self._generate_embeddings(input=[image_dict])[0]
+
+    def generate_text_embeddings(
+        self, texts: Union[List[str], np.ndarray], *args, **kwargs
+    ) -> List[np.array]:
+        return self._generate_embeddings(input=texts)
+
+    def _generate_embeddings(self, input: List, *args, **kwargs) -> List[np.array]:
+        """
+        Get the embeddings for the given texts
+
+        Parameters
+        ----------
+        texts: list[str] or np.ndarray (of str)
+            The texts to embed
+        """
+        self._init_client()
+        resp = JinaEmbeddings._session.post(  # type: ignore
+            API_URL, json={"input": input, "model": self.name}
+        ).json()
+        if "data" not in resp:
+            raise RuntimeError(resp["detail"])
+
+        embeddings = resp["data"]
+
+        # Sort resulting embeddings by index
+        sorted_embeddings = sorted(embeddings, key=lambda e: e["index"])  # type: ignore
+
+        return [result["embedding"] for result in sorted_embeddings]
+
+    def _init_client(self):
+        if JinaEmbeddings._session is None:
+            if self.api_key is None and os.environ.get("JINA_API_KEY") is None:
+                api_key_not_found_help("jina")
+            api_key = self.api_key or os.environ.get("JINA_API_KEY")
+            JinaEmbeddings._session = requests.Session()
+            JinaEmbeddings._session.headers.update(
+                {"Authorization": f"Bearer {api_key}", "Accept-Encoding": "identity"}
+            )