Bump version: 0.10.2-beta.0 → 0.10.2

Fixes #1385

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.5.2"
+current_version = "0.7.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/docs_test.yml

@@ -24,7 +24,7 @@ env:
 jobs:
   test-python:
     name: Test doc python code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "warp-ubuntu-latest-x64-4x"
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -56,7 +56,7 @@ jobs:
         for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
   test-node:
     name: Test doc nodejs code
-    runs-on: "buildjet-8vcpu-ubuntu-2204"
+    runs-on: "warp-ubuntu-latest-x64-4x"
     timeout-minutes: 60
     strategy:
       fail-fast: false

.github/workflows/npm-publish.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   node:
+    name: vectordb Typescript
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -39,6 +40,7 @@ jobs:
             node/vectordb-*.tgz
 
   node-macos:
+    name: vectordb ${{ matrix.config.arch }}
     strategy:
       matrix:
         config:
@@ -69,6 +71,7 @@ jobs:
             node/dist/lancedb-vectordb-darwin*.tgz
 
   nodejs-macos:
+    name: lancedb ${{ matrix.config.arch }}
     strategy:
       matrix:
         config:
@@ -99,7 +102,7 @@ jobs:
             nodejs/dist/*.node
 
   node-linux:
-    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    name: vectordb (${{ matrix.config.arch}}-unknown-linux-gnu)
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -139,7 +142,7 @@ jobs:
             node/dist/lancedb-vectordb-linux*.tgz
 
   nodejs-linux:
-    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
+    name: lancedb (${{ matrix.config.arch}}-unknown-linux-gnu
     runs-on: ${{ matrix.config.runner }}
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -190,6 +193,7 @@ jobs:
             !nodejs/dist/*.node
 
   node-windows:
+    name: vectordb ${{ matrix.target }}
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -223,6 +227,7 @@ jobs:
             node/dist/lancedb-vectordb-win32*.tgz
 
   nodejs-windows:
+    name: lancedb ${{ matrix.target }}
     runs-on: windows-2022
     # Only runs on tags that matches the make-release action
     if: startsWith(github.ref, 'refs/tags/v')
@@ -256,6 +261,7 @@ jobs:
             nodejs/dist/*.node
 
   release:
+    name: vectordb NPM Publish
     needs: [node, node-macos, node-linux, node-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
@@ -284,8 +290,18 @@ jobs:
           for filename in *.tgz; do
             npm publish $PUBLISH_ARGS $filename
           done
+      - name: Notify Slack Action
+        uses: ravsamhq/notify-slack-action@2.3.0
+        if: ${{ always() }}
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "{workflow} is failing"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   release-nodejs:
+    name: lancedb NPM Publish
     needs: [nodejs-macos, nodejs-linux, nodejs-windows]
     runs-on: ubuntu-latest
     # Only runs on tags that matches the make-release action
@@ -333,6 +349,15 @@ jobs:
           else
             npm publish --access public
           fi
+      - name: Notify Slack Action
+        uses: ravsamhq/notify-slack-action@2.3.0
+        if: ${{ always() }}
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "{workflow} is failing"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   update-package-lock:
     needs: [release]

.github/workflows/python.yml

@@ -33,11 +33,11 @@ jobs:
           python-version: "3.11"
       - name: Install ruff
         run: |
-          pip install ruff==0.2.2
+          pip install ruff==0.5.4
       - name: Format check
         run: ruff format --check .
       - name: Lint
-        run: ruff .
+        run: ruff check .
   doctest:
     name: "Doctest"
     timeout-minutes: 30

.github/workflows/rust.yml

@@ -53,7 +53,10 @@ jobs:
       run: cargo clippy --all --all-features -- -D warnings
   linux:
     timeout-minutes: 30
-    runs-on: ubuntu-22.04
+    # To build all features, we need more disk space than is available
+    # on the GitHub-provided runner. This is mostly due to the the
+    # sentence-transformers feature.
+    runs-on: warp-ubuntu-latest-x64-4x
     defaults:
       run:
         shell: bash
@@ -131,4 +134,3 @@ jobs:
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
           cargo build
           cargo test
-  

.gitignore

@@ -4,6 +4,7 @@
 **/__pycache__
 .DS_Store
 venv
+.venv
 
 .vscode
 .zed

.pre-commit-config.yaml

@@ -14,8 +14,8 @@ repos:
   hooks:
     - id: local-biome-check
       name: biome check
-      entry: npx @biomejs/biome@1.7.3 check --config-path nodejs/biome.json nodejs/
+      entry: npx @biomejs/biome@1.8.3 check --config-path nodejs/biome.json nodejs/
       language: system
       types: [text]
       files: "nodejs/.*"
-      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*
+      exclude: nodejs/lancedb/native.d.ts|nodejs/dist/.*|nodejs/examples/.*

Cargo.toml

@@ -20,11 +20,11 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.13.0", "features" = ["dynamodb"] }
+lance = { "version" = "=0.14.1", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.13.0" }
+lance-index = { "version" = "=0.14.1" }
-lance-linalg = { "version" = "=0.13.0" }
+lance-linalg = { "version" = "=0.14.1" }
-lance-testing = { "version" = "=0.13.0" }
+lance-testing = { "version" = "=0.14.1" }
-lance-datafusion = { "version" = "=0.13.0" }
+lance-datafusion = { "version" = "=0.14.1" }
 # Note that this one does not include pyarrow
 arrow = { version = "51.0", optional = false }
 arrow-array = "51.0"

ci/manylinux_node/Dockerfile

@@ -18,8 +18,8 @@ COPY install_protobuf.sh install_protobuf.sh
 RUN ./install_protobuf.sh ${ARCH}
 
 ENV DOCKER_USER=${DOCKER_USER}
-# Create a group and user
+# Create a group and user, but only if it doesn't exist
-RUN echo ${ARCH} && adduser --user-group --create-home --uid ${DOCKER_USER} build_user
+RUN echo ${ARCH} && id -u ${DOCKER_USER} >/dev/null 2>&1 || adduser --user-group --create-home --uid ${DOCKER_USER} build_user
 
 # We switch to the user to install Rust and Node, since those like to be 
 # installed at the user level.

docs/mkdocs.yml

@@ -57,6 +57,8 @@ plugins:
             - https://arrow.apache.org/docs/objects.inv
             - https://pandas.pydata.org/docs/objects.inv
   - mkdocs-jupyter
+  - render_swagger:
+      allow_arbitrary_locations : true
 
 markdown_extensions:
   - admonition
@@ -100,15 +102,18 @@ nav:
               - Linear Combination Reranker: reranking/linear_combination.md
               - Cross Encoder Reranker: reranking/cross_encoder.md
               - ColBERT Reranker: reranking/colbert.md
+              - Jina Reranker: reranking/jina.md
               - OpenAI Reranker: reranking/openai.md
               - Building Custom Rerankers: reranking/custom_reranker.md
+              - Example: notebooks/lancedb_reranking.ipynb
           - Filtering: sql.md
           - Versioning & Reproducibility: notebooks/reproducibility.ipynb
           - Configuring Storage: guides/storage.md
-          - Sync -> Async Migration Guide: migration.md
+          - Migration Guide: migration.md
           - Tuning retrieval performance:
               - Choosing right query type: guides/tuning_retrievers/1_query_types.md
               - Reranking: guides/tuning_retrievers/2_reranking.md
+              - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
       - 🧬 Managing embeddings:
           - Overview: embeddings/index.md
           - Embedding functions: embeddings/embedding_functions.md
@@ -123,10 +128,11 @@ nav:
           - DuckDB: python/duckdb.md
           - LangChain:
             - LangChain 🔗: integrations/langchain.md
+            - LangChain demo: notebooks/langchain_demo.ipynb
             - LangChain JS/TS 🔗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
           - LlamaIndex 🦙:
             - LlamaIndex docs: integrations/llamaIndex.md
-            - LlamaIndex demo:  https://docs.llamaindex.ai/en/stable/examples/vector_stores/LanceDBIndexDemo/
+            - LlamaIndex demo: notebooks/llamaIndex_demo.ipynb
           - Pydantic: python/pydantic.md
           - Voxel51: integrations/voxel51.md
           - PromptTools: integrations/prompttools.md
@@ -151,13 +157,14 @@ nav:
       - ⚙️ API reference:
           - 🐍 Python: python/python.md
           - 👾 JavaScript (vectordb): javascript/modules.md
-          - 👾 JavaScript (lancedb): javascript/modules.md
+          - 👾 JavaScript (lancedb): js/globals.md
           - 🦀 Rust: https://docs.rs/lancedb/latest/lancedb/
       - ☁️ LanceDB Cloud:
           - Overview: cloud/index.md
           - API reference:
               - 🐍 Python: python/saas-python.md
               - 👾 JavaScript: javascript/modules.md
+              - REST API: cloud/rest.md
 
   - Quick start: basic.md
   - Concepts:
@@ -180,15 +187,18 @@ nav:
           - Linear Combination Reranker: reranking/linear_combination.md
           - Cross Encoder Reranker: reranking/cross_encoder.md
           - ColBERT Reranker: reranking/colbert.md
+          - Jina Reranker: reranking/jina.md
           - OpenAI Reranker: reranking/openai.md
           - Building Custom Rerankers: reranking/custom_reranker.md
+          - Example: notebooks/lancedb_reranking.ipynb
       - Filtering: sql.md
       - Versioning & Reproducibility: notebooks/reproducibility.ipynb
       - Configuring Storage: guides/storage.md
-      - Sync -> Async Migration Guide: migration.md
+      - Migration Guide: migration.md
       - Tuning retrieval performance:
           - Choosing right query type: guides/tuning_retrievers/1_query_types.md
           - Reranking: guides/tuning_retrievers/2_reranking.md
+          - Embedding fine-tuning: guides/tuning_retrievers/3_embed_tuning.md
   - Managing Embeddings:
       - Overview: embeddings/index.md
       - Embedding functions: embeddings/embedding_functions.md
@@ -201,9 +211,9 @@ nav:
       - Pandas and PyArrow: python/pandas_and_pyarrow.md
       - Polars: python/polars_arrow.md
       - DuckDB: python/duckdb.md
-      - LangChain 🦜️🔗↗: https://python.langchain.com/docs/integrations/vectorstores/lancedb
+      - LangChain 🦜️🔗↗: integrations/langchain.md
       - LangChain.js 🦜️🔗↗: https://js.langchain.com/docs/integrations/vectorstores/lancedb
-      - LlamaIndex 🦙↗: https://gpt-index.readthedocs.io/en/latest/examples/vector_stores/LanceDBIndexDemo.html
+      - LlamaIndex 🦙↗: integrations/llamaIndex.md
       - Pydantic: python/pydantic.md
       - Voxel51: integrations/voxel51.md
       - PromptTools: integrations/prompttools.md
@@ -221,13 +231,14 @@ nav:
       - Overview: api_reference.md
       - Python: python/python.md
       - Javascript (vectordb): javascript/modules.md
-      - Javascript (lancedb): js/modules.md
+      - Javascript (lancedb): js/globals.md
       - Rust: https://docs.rs/lancedb/latest/lancedb/index.html
   - LanceDB Cloud:
       - Overview: cloud/index.md
       - API reference:
           - 🐍 Python: python/saas-python.md
           - 👾 JavaScript: javascript/modules.md
+          - REST API: cloud/rest.md
 
 extra_css:
   - styles/global.css

docs/openapi.yml

@@ -0,0 +1,487 @@
+openapi: 3.1.0
+info:
+  version: 1.0.0
+  title: LanceDB Cloud API
+  description: |
+    LanceDB Cloud API is a RESTful API that allows users to access and modify data stored in LanceDB Cloud.
+    Table actions are considered temporary resource creations and all use POST method.
+  contact:
+    name: LanceDB support
+    url: https://lancedb.com
+    email: contact@lancedb.com
+
+servers:
+  - url: https://{db}.{region}.api.lancedb.com
+    description: LanceDB Cloud REST endpoint.
+    variables:
+      db:
+        default: ""
+        description: the name of DB
+      region:
+        default: "us-east-1"
+        description: the service region of the DB
+
+security:
+  - key_auth: []
+
+components:
+  securitySchemes:
+    key_auth:
+      name: x-api-key
+      type: apiKey
+      in: header
+  parameters:
+    table_name:
+      name: name
+      in: path
+      description: name of the table
+      required: true
+      schema:
+        type: string
+  responses:
+    invalid_request:
+      description: Invalid request
+      content:
+        text/plain:
+          schema:
+            type: string
+    not_found:
+      description: Not found
+      content:
+        text/plain:
+          schema:
+            type: string
+    unauthorized:
+      description: Unauthorized
+      content:
+        text/plain:
+          schema:
+            type: string
+  requestBodies:
+    arrow_stream_buffer:
+      description: Arrow IPC stream buffer
+      required: true
+      content:
+        application/vnd.apache.arrow.stream:
+          schema:
+            type: string
+            format: binary
+
+paths:
+  /v1/table/:
+    get:
+      description: List tables, optionally, with pagination.
+      tags:
+        - Tables
+      summary: List Tables
+      operationId: listTables
+      parameters:
+        - name: limit
+          in: query
+          description: Limits the number of items to return.
+          schema:
+            type: integer
+        - name: page_token
+          in: query
+          description: Specifies the starting position of the next query
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Successfully returned a list of tables in the DB
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  tables:
+                    type: array
+                    items:
+                      type: string
+                  page_token:
+                    type: string
+
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/create/:
+    post:
+      description: Create a new table
+      summary: Create a new table
+      operationId: createTable
+      tags:
+        - Tables
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Table successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/query/:
+    post:
+      description: Vector Query
+      url: https://{db-uri}.{aws-region}.api.lancedb.com/v1/table/{name}/query/
+      tags:
+        - Data
+      summary: Vector Query
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                vector:
+                  type: FixedSizeList
+                  description: |
+                    The targetted vector to search for. Required.
+                vector_column:
+                  type: string
+                  description: |
+                    The column to query, it can be inferred from the schema if there is only one vector column.
+                prefilter:
+                  type: boolean
+                  description: |
+                    Whether to prefilter the data. Optional.
+                k:
+                  type: integer
+                  description: |
+                    The number of search results to return. Default is 10.
+                distance_type:
+                  type: string
+                  description: |
+                    The distance metric to use for search. L2, Cosine, Dot and Hamming are supported. Default is L2.
+                bypass_vector_index:
+                  type: boolean
+                  description: |
+                    Whether to bypass vector index. Optional.
+                filter:
+                  type: string
+                  description: |
+                    A filter expression that specifies the rows to query. Optional.
+                columns:
+                  type: array
+                  items:
+                    type: string
+                  description: |
+                    The columns to return. Optional.
+                nprobe:
+                  type: integer
+                  description: |
+                    The number of probes to use for search. Optional.
+                refine_factor:
+                  type: integer
+                  description: |
+                    The refine factor to use for search. Optional.
+                  default: null
+                fast_search:
+                  type: boolean
+                  description: |
+                    Whether to use fast search. Optional.
+                  default: false
+              required:
+                - vector
+
+      responses:
+        "200":
+          description: top k results if query is successfully executed
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  results:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        id:
+                          type: integer
+                        selected_col_1_to_return:
+                          type: col_1_type
+                        selected_col_n_to_return:
+                          type: col_n_type
+                        _distance:
+                          type: float
+
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/insert/:
+    post:
+      description: Insert new data to the Table.
+      tags:
+        - Data
+      operationId: insertData
+      summary: Insert new data.
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Insert successful
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/merge_insert/:
+    post:
+      description: Create a "merge insert" operation
+        This operation can add rows, update rows, and remove rows all in a single
+        transaction. See python method `lancedb.table.Table.merge_insert` for examples.
+      tags:
+        - Data
+      summary: Merge Insert
+      operationId: mergeInsert
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+        - name: on
+          in: query
+          description: |
+            The column to use as the primary key for the merge operation.
+          required: true
+          schema:
+            type: string
+        - name: when_matched_update_all
+          in: query
+          description: |
+            Rows that exist in both the source table (new data) and
+            the target table (old data) will be updated, replacing
+            the old row with the corresponding matching row.
+          required: false
+          schema:
+            type: boolean
+        - name: when_matched_update_all_filt
+          in: query
+          description: |
+            If present then only rows that satisfy the filter expression will
+            be updated
+          required: false
+          schema:
+            type: string
+        - name: when_not_matched_insert_all
+          in: query
+          description: |
+            Rows that exist only in the source table (new data) will be
+            inserted into the target table (old data).
+          required: false
+          schema:
+            type: boolean
+        - name: when_not_matched_by_source_delete
+          in: query
+          description: |
+            Rows that exist only in the target table (old data) will be
+            deleted. An optional condition (`when_not_matched_by_source_delete_filt`)
+            can be provided to limit what data is deleted.
+          required: false
+          schema:
+            type: boolean
+        - name: when_not_matched_by_source_delete_filt
+          in: query
+          description: |
+            The filter expression that specifies the rows to delete.
+          required: false
+          schema:
+            type: string
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Merge Insert successful
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/delete/:
+    post:
+      description: Delete rows from a table.
+      tags:
+        - Data
+      summary: Delete rows from a table
+      operationId: deleteData
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                predicate:
+                  type: string
+                  description: |
+                    A filter expression that specifies the rows to delete.
+      responses:
+        "200":
+          description: Delete successful
+        "401":
+          $ref: "#/components/responses/unauthorized"
+  /v1/table/{name}/drop/:
+    post:
+      description: Drop a table
+      tags:
+        - Tables
+      summary: Drop a table
+      operationId: dropTable
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        $ref: "#/components/requestBodies/arrow_stream_buffer"
+      responses:
+        "200":
+          description: Drop successful
+        "401":
+          $ref: "#/components/responses/unauthorized"
+
+  /v1/table/{name}/describe/:
+    post:
+      description: Describe a table and return Table Information.
+      tags:
+        - Tables
+      summary: Describe a table
+      operationId: describeTable
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      responses:
+        "200":
+          description: Table information
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  table:
+                    type: string
+                  version:
+                    type: integer
+                  schema:
+                    type: string
+                  stats:
+                    type: object
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+
+  /v1/table/{name}/index/list/:
+    post:
+      description: List indexes of a table
+      tags:
+        - Tables
+      summary: List indexes of a table
+      operationId: listIndexes
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      responses:
+        "200":
+          description: Available list of indexes on the table.
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  indexes:
+                    type: array
+                    items:
+                      type: object
+                      properties:
+                        columns:
+                          type: array
+                          items:
+                            type: string
+                        index_name:
+                          type: string
+                        index_uuid:
+                          type: string
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/create_index/:
+    post:
+      description: Create vector index on a Table
+      tags:
+        - Tables
+      summary: Create vector index on a Table
+      operationId: createIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                column:
+                  type: string
+                metric_type:
+                  type: string
+                  nullable: false
+                  description: |
+                    The metric type to use for the index. L2, Cosine, Dot are supported.
+                index_type:
+                  type: string
+      responses:
+        "200":
+          description: Index successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"
+  /v1/table/{name}/create_scalar_index/:
+    post:
+      description: Create a scalar index on a table
+      tags:
+        - Tables
+      summary: Create a scalar index on a table
+      operationId: createScalarIndex
+      parameters:
+        - $ref: "#/components/parameters/table_name"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                column:
+                  type: string
+                index_type:
+                  type: string
+                  required: false
+      responses:
+        "200":
+          description: Scalar Index successfully created
+        "400":
+          $ref: "#/components/responses/invalid_request"
+        "401":
+          $ref: "#/components/responses/unauthorized"
+        "404":
+          $ref: "#/components/responses/not_found"

docs/requirements.txt

@@ -2,4 +2,5 @@ mkdocs==1.5.3
 mkdocs-jupyter==0.24.1
 mkdocs-material==9.5.3
 mkdocstrings[python]==0.20.0
+mkdocs-render-swagger-plugin
 pydantic

docs/src/ann_indexes.md

@@ -38,13 +38,27 @@ Lance supports `IVF_PQ` index type by default.
     tbl.create_index(num_partitions=256, num_sub_vectors=96)
     ```
 
-=== "Typescript"
+=== "TypeScript"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<--- "docs/src/ann_indexes.ts:import"
 
-    --8<-- "docs/src/ann_indexes.ts:ingest"
+        Creating indexes is done via the [lancedb.Table.createIndex](../js/classes/Table.md/#createIndex) method.
-    ```
+
+        ```typescript
+        --8<--- "nodejs/examples/ann_indexes.ts:import"
+
+        --8<-- "nodejs/examples/ann_indexes.ts:ingest"
+        ```
+
+    === "vectordb (deprecated)"
+
+        Creating indexes is done via the [lancedb.Table.createIndex](../javascript/interfaces/Table.md/#createIndex) method.
+
+        ```typescript
+        --8<--- "docs/src/ann_indexes.ts:import"
+
+        --8<-- "docs/src/ann_indexes.ts:ingest"
+        ```
 
 === "Rust"
 
@@ -91,27 +105,27 @@ You can specify the GPU device to train IVF partitions via
 
 === "Linux"
 
-     <!-- skip-test -->
+    <!-- skip-test -->
-     ``` { .python .copy }
+    ``` { .python .copy }
-     # Create index using CUDA on Nvidia GPUs.
+    # Create index using CUDA on Nvidia GPUs.
-     tbl.create_index(
+    tbl.create_index(
-          num_partitions=256,
+        num_partitions=256,
-          num_sub_vectors=96,
+        num_sub_vectors=96,
-          accelerator="cuda"
+        accelerator="cuda"
-     )
+    )
-     ```
+    ```
 
 === "MacOS"
 
-     <!-- skip-test -->
+    <!-- skip-test -->
-     ```python
+    ```python
-     # Create index using MPS on Apple Silicon.
+    # Create index using MPS on Apple Silicon.
-     tbl.create_index(
+    tbl.create_index(
-          num_partitions=256,
+        num_partitions=256,
-          num_sub_vectors=96,
+        num_sub_vectors=96,
-          accelerator="mps"
+        accelerator="mps"
-     )
+    )
-     ```
+    ```
 
 Troubleshooting:
 
@@ -150,11 +164,19 @@ There are a couple of parameters that can be used to fine-tune the search:
     1  [0.48587373, 0.269207, 0.15095535, 0.65531915,...  item 3953  108.393867
     ```
 
-=== "Typescript"
+=== "TypeScript"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/ann_indexes.ts:search1"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search1"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/ann_indexes.ts:search1"
+        ```
 
 === "Rust"
 
@@ -172,15 +194,23 @@ You can further filter the elements returned by a search using a where clause.
 
 === "Python"
 
-     ```python
+    ```python
-     tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
+    tbl.search(np.random.random((1536))).where("item != 'item 1141'").to_pandas()
-     ```
+    ```
 
-=== "Typescript"
+=== "TypeScript"
 
-     ```javascript
+    === "@lancedb/lancedb"
-     --8<-- "docs/src/ann_indexes.ts:search2"
+
-     ```
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search2"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```javascript
+        --8<-- "docs/src/ann_indexes.ts:search2"
+        ```
 
 ### Projections (select clause)
 
@@ -188,23 +218,31 @@ You can select the columns returned by the query using a select clause.
 
 === "Python"
 
-     ```python
+    ```python
-     tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
+    tbl.search(np.random.random((1536))).select(["vector"]).to_pandas()
-     ```
+    ```
 
 
-     ```text
+    ```text
-                                                   vector _distance
+                                                vector _distance
-     0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
+    0  [0.30928212, 0.022668175, 0.1756372, 0.4911822...  93.971092
-     1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
+    1  [0.2525465, 0.01723831, 0.261568, 0.002007689,...  95.173485
-     ...
+    ...
-     ```
+    ```
 
-=== "Typescript"
+=== "TypeScript"
 
-     ```typescript
+    === "@lancedb/lancedb"
-     --8<-- "docs/src/ann_indexes.ts:search3"
+
-     ```
+        ```typescript
+        --8<-- "nodejs/examples/ann_indexes.ts:search3"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/ann_indexes.ts:search3"
+        ```
 
 ## FAQ
 

docs/src/api_reference.md

@@ -4,5 +4,5 @@ The API reference for the LanceDB client SDKs are available at the following loc
 
 - [Python](python/python.md)
 - [JavaScript (legacy vectordb package)](javascript/modules.md)
-- [JavaScript (newer @lancedb/lancedb package)](js/modules.md)
+- [JavaScript (newer @lancedb/lancedb package)](js/globals.md)
 - [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)

docs/src/basic.md

@@ -16,11 +16,60 @@
       pip install lancedb
       ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-      ```shell
+        ```shell
-      npm install vectordb
+        npm install @lancedb/lancedb
-      ```
+        ```
+        !!! note "Bundling `@lancedb/lancedb` apps with Webpack"
+
+            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+            ```javascript
+            /** @type {import('next').NextConfig} */
+            module.exports = ({
+            webpack(config) {
+                config.externals.push({ '@lancedb/lancedb': '@lancedb/lancedb' })
+                return config;
+            }
+            })
+            ```
+
+        !!! note "Yarn users"
+
+            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
+
+            ```shell
+            yarn add apache-arrow
+            ```
+
+    === "vectordb (deprecated)"
+
+        ```shell
+        npm install vectordb
+        ```
+        !!! note "Bundling `vectordb` apps with Webpack"
+
+            Since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
+
+            ```javascript
+            /** @type {import('next').NextConfig} */
+            module.exports = ({
+            webpack(config) {
+                config.externals.push({ vectordb: 'vectordb' })
+                return config;
+            }
+            })
+            ```
+
+        !!! note "Yarn users"
+
+            Unlike other package managers, Yarn does not automatically resolve peer dependencies. If you are using Yarn, you will need to manually install 'apache-arrow':
+
+            ```shell
+            yarn add apache-arrow
+            ```
 
 === "Rust"
 
@@ -58,11 +107,18 @@ recommend switching to stable releases.
       pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
       ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-      ```shell
+    === "@lancedb/lancedb"
-      npm install vectordb@preview
+
-      ```
+        ```shell
+        npm install @lancedb/lancedb@preview
+        ```
+    === "vectordb (deprecated)"
+
+        ```shell
+        npm install vectordb@preview
+        ```
 
 === "Rust"
 
@@ -93,23 +149,22 @@ recommend switching to stable releases.
         use the same syntax as the asynchronous API.  To help with this migration we
         have created a [migration guide](migration.md) detailing the differences.
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:import"
 
-    --8<-- "docs/src/basic_legacy.ts:open_db"
+        ```typescript
-    ```
+        import * as lancedb from "@lancedb/lancedb";
+        import * as arrow from "apache-arrow";
 
-    !!! note "`@lancedb/lancedb` vs. `vectordb`"
+        --8<-- "nodejs/examples/basic.ts:connect"
+        ```
 
-        The Javascript SDK was originally released as `vectordb`.  In an effort to
+    === "vectordb (deprecated)"
-        reduce maintenance we are aligning our SDKs.  The new, aligned, Javascript
+
-        API is being released as `lancedb`.  If you are starting new work we encourage
+        ```typescript
-        you to try out `lancedb`.  Once the new API is feature complete we will begin
+        --8<-- "docs/src/basic_legacy.ts:open_db"
-        slowly deprecating `vectordb` in favor of `lancedb`. There is a
+        ```
-        [migration guide](migration.md) detailing the differences which will assist
-        you in this process.
 
 === "Rust"
 
@@ -152,15 +207,23 @@ table.
     --8<-- "python/python/tests/docs/test_basic.py:create_table_async_pandas"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:create_table"
-    ```
 
-    If the table already exists, LanceDB will raise an error by default.
+        ```typescript
-    If you want to overwrite the table, you can pass in `mode="overwrite"`
+        --8<-- "nodejs/examples/basic.ts:create_table"
-    to the `createTable` function.
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_table"
+        ```
+
+        If the table already exists, LanceDB will raise an error by default.
+        If you want to overwrite the table, you can pass in `mode:"overwrite"`
+        to the `createTable` function.
 
 === "Rust"
 
@@ -200,11 +263,19 @@ similar to a `CREATE TABLE` statement in SQL.
     !!! note "You can define schema in Pydantic"
         LanceDB comes with Pydantic support, which allows you to define the schema of your data using Pydantic models. This makes it easy to work with LanceDB tables and data. Learn more about all supported types in [tables guide](./guides/tables.md).
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+        ```
 
 === "Rust"
 
@@ -223,11 +294,19 @@ Once created, you can open a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:open_table_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:open_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        const tbl = await db.openTable("myTable");
+        ```
 
-    ```typescript
-    const tbl = await db.openTable("myTable");
-    ```
 
 === "Rust"
 
@@ -244,11 +323,18 @@ If you forget the name of your table, you can always get a listing of all table
     --8<-- "python/python/tests/docs/test_basic.py:table_names_async"
     ```
 
-=== "Javascript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```javascript
+        ```typescript
-    console.log(await db.tableNames());
+        --8<-- "nodejs/examples/basic.ts:table_names"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        console.log(await db.tableNames());
+        ```
 
 === "Rust"
 
@@ -267,11 +353,18 @@ After a table has been created, you can always add more data to it as follows:
     --8<-- "python/python/tests/docs/test_basic.py:add_data_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```typescript
+        ```typescript
-    --8<-- "docs/src/basic_legacy.ts:add"
+        --8<-- "nodejs/examples/basic.ts:add_data"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:add"
+        ```
 
 === "Rust"
 
@@ -292,11 +385,18 @@ Once you've embedded the query, you can find its nearest neighbors as follows:
 
     This returns a pandas DataFrame with the results.
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```typescript
+        ```typescript
-    --8<-- "docs/src/basic_legacy.ts:search"
+        --8<-- "nodejs/examples/basic.ts:vector_search"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:search"
+        ```
 
 === "Rust"
 
@@ -325,11 +425,18 @@ LanceDB allows you to create an ANN index on a table as follows:
     --8<-- "python/python/tests/docs/test_basic.py:create_index_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
+    === "@lancedb/lancedb"
 
-    ```{.typescript .ignore}
+        ```typescript
-    --8<-- "docs/src/basic_legacy.ts:create_index"
+        --8<-- "nodejs/examples/basic.ts:create_index"
-    ```
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```{.typescript .ignore}
+        --8<-- "docs/src/basic_legacy.ts:create_index"
+        ```
 
 === "Rust"
 
@@ -357,11 +464,19 @@ This can delete any number of rows that match the filter.
     --8<-- "python/python/tests/docs/test_basic.py:delete_rows_async"
     ```
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-    ```typescript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:delete"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:delete_rows"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:delete"
+        ```
 
 === "Rust"
 
@@ -378,9 +493,15 @@ simple or complex as needed. To see what expressions are supported, see the
 
       Read more: [lancedb.table.Table.delete][]
 
-=== "Javascript"
+=== "Typescript[^1]"
 
-      Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
+    === "@lancedb/lancedb"
+
+        Read more: [lancedb.Table.delete](javascript/interfaces/Table.md#delete)
+
+    === "vectordb (deprecated)"
+
+        Read more: [vectordb.Table.delete](javascript/interfaces/Table.md#delete)
 
 === "Rust"
 
@@ -392,23 +513,31 @@ Use the `drop_table()` method on the database to remove a table.
 
 === "Python"
 
-      ```python
+    ```python
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table"
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table"
-      --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
+    --8<-- "python/python/tests/docs/test_basic.py:drop_table_async"
-      ```
+    ```
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
+    This permanently removes the table and is not recoverable, unlike deleting rows.
-      By default, if the table does not exist an exception is raised. To suppress this,
+    By default, if the table does not exist an exception is raised. To suppress this,
-      you can pass in `ignore_missing=True`.
+    you can pass in `ignore_missing=True`.
 
-=== "Typescript"
+=== "Typescript[^1]"
 
-      ```typescript
+    === "@lancedb/lancedb"
-      --8<-- "docs/src/basic_legacy.ts:drop_table"
-      ```
 
-      This permanently removes the table and is not recoverable, unlike deleting rows.
+        ```typescript
-      If the table does not exist an exception is raised.
+        --8<-- "nodejs/examples/basic.ts:drop_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:drop_table"
+        ```
+
+        This permanently removes the table and is not recoverable, unlike deleting rows.
+        If the table does not exist an exception is raised.
 
 === "Rust"
 
@@ -416,19 +545,6 @@ Use the `drop_table()` method on the database to remove a table.
     --8<-- "rust/lancedb/examples/simple.rs:drop_table"
     ```
 
-!!! note "Bundling `vectordb` apps with Webpack"
-
-    If you're using the `vectordb` module in JavaScript, since LanceDB contains a prebuilt Node binary, you must configure `next.config.js` to exclude it from webpack. This is required for both using Next.js and deploying a LanceDB app on Vercel.
-
-    ```javascript
-    /** @type {import('next').NextConfig} */
-    module.exports = ({
-    webpack(config) {
-        config.externals.push({ vectordb: 'vectordb' })
-        return config;
-    }
-    })
-    ```
 
 ## Using the Embedding API
 You can use the embedding API when working with embedding models. It automatically vectorizes the data at ingestion and query time and comes with built-in integrations with popular embedding models like Openai, Hugging Face, Sentence Transformers, CLIP and more.
@@ -440,6 +556,22 @@ You can use the embedding API when working with embedding models. It automatical
     --8<-- "python/python/tests/docs/test_embeddings_optional.py:openai_embeddings"
     ```
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:openai_embeddings"
+        ```
+
+=== "Rust"
+
+    ```rust
+    --8<-- "rust/lancedb/examples/openai.rs:imports"
+    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    ```
+
 Learn about using the existing integrations and creating custom embedding functions in the [embedding API guide](./embeddings/).
 
 
@@ -448,3 +580,5 @@ Learn about using the existing integrations and creating custom embedding functi
 This section covered the very basics of using LanceDB. If you're learning about vector databases for the first time, you may want to read the page on [indexing](concepts/index_ivfpq.md) to get familiar with the concepts.
 
 If you've already worked with other vector databases, you may want to read the [guides](guides/tables.md) to learn how to work with LanceDB in more detail.
+
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/basic_legacy.ts

@@ -1,6 +1,14 @@
 // --8<-- [start:import]
 import * as lancedb from "vectordb";
-import { Schema, Field, Float32, FixedSizeList, Int32, Float16 } from "apache-arrow";
+import {
+  Schema,
+  Field,
+  Float32,
+  FixedSizeList,
+  Int32,
+  Float16,
+} from "apache-arrow";
+import * as arrow from "apache-arrow";
 // --8<-- [end:import]
 import * as fs from "fs";
 import { Table as ArrowTable, Utf8 } from "apache-arrow";
@@ -20,9 +28,33 @@ const example = async () => {
       { vector: [3.1, 4.1], item: "foo", price: 10.0 },
       { vector: [5.9, 26.5], item: "bar", price: 20.0 },
     ],
-    { writeMode: lancedb.WriteMode.Overwrite }
+    { writeMode: lancedb.WriteMode.Overwrite },
   );
   // --8<-- [end:create_table]
+  {
+    // --8<-- [start:create_table_with_schema]
+    const schema = new arrow.Schema([
+      new arrow.Field(
+        "vector",
+        new arrow.FixedSizeList(
+          2,
+          new arrow.Field("item", new arrow.Float32(), true),
+        ),
+      ),
+      new arrow.Field("item", new arrow.Utf8(), true),
+      new arrow.Field("price", new arrow.Float32(), true),
+    ]);
+    const data = [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ];
+    const tbl = await db.createTable({
+      name: "myTableWithSchema",
+      data,
+      schema,
+    });
+    // --8<-- [end:create_table_with_schema]
+  }
 
   // --8<-- [start:add]
   const newData = Array.from({ length: 500 }, (_, i) => ({
@@ -42,33 +74,35 @@ const example = async () => {
   // --8<-- [end:create_index]
 
   // --8<-- [start:create_empty_table]
-  const schema = new Schema([
+  const schema = new arrow.Schema([
-    new Field("id", new Int32()),
+    new arrow.Field("id", new arrow.Int32()),
-    new Field("name", new Utf8()),
+    new arrow.Field("name", new arrow.Utf8()),
   ]);
+
   const empty_tbl = await db.createTable({ name: "empty_table", schema });
   // --8<-- [end:create_empty_table]
-
+  {
-  // --8<-- [start:create_f16_table]
+    // --8<-- [start:create_f16_table]
-  const dim = 16
+    const dim = 16;
-  const total = 10
+    const total = 10;
-  const f16_schema = new Schema([
+    const schema = new Schema([
-      new Field('id', new Int32()),
+      new Field("id", new Int32()),
       new Field(
-        'vector',
+        "vector",
-        new FixedSizeList(dim, new Field('item', new Float16(), true)),
+        new FixedSizeList(dim, new Field("item", new Float16(), true)),
-        false
+        false,
-      )
+      ),
-    ])
+    ]);
-  const data = lancedb.makeArrowTable(
+    const data = lancedb.makeArrowTable(
       Array.from(Array(total), (_, i) => ({
         id: i,
-        vector: Array.from(Array(dim), Math.random)
+        vector: Array.from(Array(dim), Math.random),
       })),
-      { f16_schema }
+      { schema },
-    )
+    );
-  const table = await db.createTable('f16_tbl', data)
+    const table = await db.createTable("f16_tbl", data);
-  // --8<-- [end:create_f16_table]
+    // --8<-- [end:create_f16_table]
+  }
 
   // --8<-- [start:search]
   const query = await tbl.search([100, 100]).limit(2).execute();

docs/src/cloud/rest.md

@@ -0,0 +1 @@
+!!swagger ../../openapi.yml!!

docs/src/embeddings/default_embedding_functions.md

@@ -17,6 +17,7 @@ Allows you to set parameters when registering a `sentence-transformers` object.
 | `name` | `str` | `all-MiniLM-L6-v2` | The name of the model |
 | `device` | `str` | `cpu` | The device to run the model on (can be `cpu` or `gpu`) |
 | `normalize` | `bool` | `True` | Whether to normalize the input text before feeding it to the model |
+| `trust_remote_code` | `bool` | `False` | Whether to trust and execute remote code from the model's Huggingface repository |
 
 
 ??? "Check out available sentence-transformer models here!"
@@ -193,13 +194,13 @@ from lancedb.pydantic import LanceModel, Vector
 
 model = get_registry().get("huggingface").create(name='facebook/bart-base')
 
-class TextModel(LanceModel):
+class Words(LanceModel):
     text: str = model.SourceField()
     vector: Vector(model.ndims()) = model.VectorField()
 
 df = pd.DataFrame({"text": ["hi hello sayonara", "goodbye world"]})
 table = db.create_table("greets", schema=Words)
-table.add()
+table.add(df)
 query = "old greeting"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
@@ -427,6 +428,45 @@ Usage Example:
         tbl.add(data)
     ```
 
+### Jina Embeddings
+Jina embeddings are used to generate embeddings for text and image data.
+You also need to set the `JINA_API_KEY` environment variable to use the Jina API.
+
+You can find a list of supported models under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+```python
+    import os
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import EmbeddingFunctionRegistry
+
+    os.environ['JINA_API_KEY'] = 'jina_*'
+
+    jina_embed = EmbeddingFunctionRegistry.get_instance().get("jina").create(name="jina-embeddings-v2-base-en")
+
+
+    class TextModel(LanceModel):
+        text: str = jina_embed.SourceField()
+        vector: Vector(jina_embed.ndims()) = jina_embed.VectorField()
+
+
+    data = [{"text": "hello world"},
+            {"text": "goodbye world"}]
+
+    db = lancedb.connect("~/.lancedb-2")
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(data)
+```
+
 ### AWS Bedrock Text Embedding Functions
 AWS Bedrock supports multiple base models for generating text embeddings. You need to setup the AWS credentials to use this embedding function.
 You can do so by using `awscli` and also add your session_token:
@@ -524,7 +564,7 @@ uris = [
 # get each uri as bytes
 image_bytes = [requests.get(uri).content for uri in uris]
 table.add(
-    [{"label": labels, "image_uri": uris, "image_bytes": image_bytes}]
+    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
 )
 ```
 Now we can search using text from both the default vector column and the custom vector column
@@ -630,3 +670,54 @@ print(actual.text == "bird")
 ```
 
 If you have any questions about the embeddings API, supported models, or see a relevant model missing, please raise an issue [on GitHub](https://github.com/lancedb/lancedb/issues).
+
+### Jina Embeddings
+Jina embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
+under [https://jina.ai/embeddings/](https://jina.ai/embeddings/)
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value | Description |
+|---|---|---|---|
+| `name` | `str` | `"jina-clip-v1"` | The model ID of the jina model to use |
+
+Usage Example:
+
+```python
+    import os
+    import requests
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+    import pandas as pd
+
+    os.environ['JINA_API_KEY'] = 'jina_*'
+
+    db = lancedb.connect("~/.lancedb")
+    func = get_registry().get("jina").create()
+
+
+    class Images(LanceModel):
+        label: str
+        image_uri: str = func.SourceField()  # image uri as the source
+        image_bytes: bytes = func.SourceField()  # image bytes as the source
+        vector: Vector(func.ndims()) = func.VectorField()  # vector column
+        vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
+
+
+    table = db.create_table("images", schema=Images)
+    labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+    uris = [
+        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+        "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+        "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+        "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+        "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+    ]
+    # get each uri as bytes
+    image_bytes = [requests.get(uri).content for uri in uris]
+    table.add(
+      pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": image_bytes})
+    )
+```

docs/src/embeddings/embedding_functions.md

@@ -29,17 +29,32 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
     You can also define your own embedding function by implementing the `EmbeddingFunction`
     abstract base interface. It subclasses Pydantic Model which can be utilized to write complex schemas simply as we'll see next!
 
-=== "JavaScript""
+=== "TypeScript"
     In the TypeScript SDK, the choices are more limited. For now, only the OpenAI
     embedding function is available.
 
     ```javascript
-    const lancedb = require("vectordb");
+    import * as lancedb from '@lancedb/lancedb'
+    import { getRegistry } from '@lancedb/lancedb/embeddings'
 
     // You need to provide an OpenAI API key
     const apiKey = "sk-..."
     // The embedding function will create embeddings for the 'text' column
-    const embedding = new lancedb.OpenAIEmbeddingFunction('text', apiKey)
+    const func = getRegistry().get("openai").create({apiKey})
+    ```
+=== "Rust"
+    In the Rust SDK, the choices are more limited. For now, only the OpenAI
+    embedding function is available. But unlike the Python and TypeScript SDKs, you need manually register the OpenAI embedding function.
+
+    ```toml
+    // Make sure to include the `openai` feature
+    [dependencies]
+    lancedb = {version = "*", features = ["openai"]}
+    ```
+
+    ```rust
+    --8<-- "rust/lancedb/examples/openai.rs:imports"
+    --8<-- "rust/lancedb/examples/openai.rs:openai_embeddings"
     ```
 
 ## 2. Define the data model or schema
@@ -55,7 +70,7 @@ For this purpose, LanceDB introduces an **embedding functions API**, that allow
 
     `VectorField` tells LanceDB to use the clip embedding function to generate query embeddings for the `vector` column and `SourceField` ensures that when adding data, we automatically use the specified embedding function to encode `image_uri`.
 
-=== "JavaScript"
+=== "TypeScript"
 
     For the TypeScript SDK, a schema can be inferred from input data, or an explicit
     Arrow schema can be provided.
@@ -74,17 +89,26 @@ the embeddings at all:
     table.add([{"image_uri": u} for u in uris])
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const db = await lancedb.connect("data/sample-lancedb");
-    const data = [
-    { text: "pepperoni"},
-    { text: "pineapple"}
-    ]
 
-    const table = await db.createTable("vectors", data, embedding)
+        ```ts
-    ```
+        --8<-- "nodejs/examples/embedding.ts:imports"
+        --8<-- "nodejs/examples/embedding.ts:embedding_function"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const db = await lancedb.connect("data/sample-lancedb");
+        const data = [
+            { text: "pepperoni"},
+            { text: "pineapple"}
+        ]
+
+        const table = await db.createTable("vectors", data, embedding)
+        ```
 
 ## 4. Querying your table
 Not only can you forget about the embeddings during ingestion, you also don't
@@ -97,8 +121,8 @@ need to worry about it when you query the table:
     ```python
     results = (
         table.search("dog")
-        .limit(10)
+            .limit(10)
-        .to_pandas()
+            .to_pandas()
     )
     ```
 
@@ -109,21 +133,31 @@ need to worry about it when you query the table:
     query_image = Image.open(p)
     results = (
         table.search(query_image)
-        .limit(10)
+            .limit(10)
-        .to_pandas()
+            .to_pandas()
     )
     ```
 
     Both of the above snippet returns a pandas DataFrame with the 10 closest vectors to the query.
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const results = await table
+
-    .search("What's the best pizza topping?")
+        ```ts
-    .limit(10)
+        const results = await table.search("What's the best pizza topping?")
-    .execute()
+            .limit(10)
-    ```    
+            .toArray()
+        ```
+
+    === "vectordb (deprecated)
+
+        ```ts
+        const results = await table
+            .search("What's the best pizza topping?")
+            .limit(10)
+            .execute()
+        ```
 
     The above snippet returns an array of records with the top 10 nearest neighbors to the query.
 

docs/src/embeddings/index.md

@@ -7,7 +7,7 @@ LanceDB supports 3 methods of working with embeddings.
 
 1. You can manually generate embeddings for the data and queries. This is done outside of LanceDB.
 2. You can use the built-in [embedding functions](./embedding_functions.md) to embed the data and queries in the background.
-3. For python users, you can define your own [custom embedding function](./custom_embedding_function.md)
+3. You can define your own [custom embedding function](./custom_embedding_function.md)
    that extends the default embedding functions.
 
 For python users, there is also a legacy [with_embeddings API](./legacy.md).
@@ -18,15 +18,103 @@ It is retained for compatibility and will be removed in a future version.
 To get started with embeddings, you can use the built-in embedding functions.
 
 ### OpenAI Embedding function
+
 LanceDB registers the OpenAI embeddings function in the registry as `openai`. You can pass any supported model name to the `create`. By default it uses `"text-embedding-ada-002"`.
 
+=== "Python"
+
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    func = get_registry().get("openai").create(name="text-embedding-ada-002")
+
+    class Words(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    table = db.create_table("words", schema=Words, mode="overwrite")
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+        )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+
+=== "TypeScript"
+
+    ```typescript
+    --8<--- "nodejs/examples/embedding.ts:imports"
+    --8<--- "nodejs/examples/embedding.ts:openai_embeddings"
+    ```
+
+=== "Rust"
+
+    ```rust
+    --8<--- "rust/lancedb/examples/openai.rs:imports"
+    --8<--- "rust/lancedb/examples/openai.rs:openai_embeddings"
+    ```
+
+### Sentence Transformers Embedding function
+LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
+
+=== "Python"
+    ```python
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry
+
+    db = lancedb.connect("/tmp/db")
+    model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
+
+    class Words(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    table = db.create_table("words", schema=Words)
+    table.add(
+        [
+            {"text": "hello world"},
+            {"text": "goodbye world"}
+        ]
+    )
+
+    query = "greetings"
+    actual = table.search(query).limit(1).to_pydantic(Words)[0]
+    print(actual.text)
+    ```
+
+=== "TypeScript"
+
+    Coming Soon!
+
+=== "Rust"
+
+    Coming Soon!
+
+### Jina Embeddings
+
+LanceDB registers the JinaAI embeddings function in the registry as `jina`. You can pass any supported model name to the `create`. By default it uses `"jina-clip-v1"`.
+`jina-clip-v1` can handle both text and images and other models only support `text`.
+
+You need to pass `JINA_API_KEY` in the environment variable or pass it as `api_key` to `create` method.
+
 ```python
+import os
 import lancedb
 from lancedb.pydantic import LanceModel, Vector
 from lancedb.embeddings import get_registry
+os.environ['JINA_API_KEY'] = "jina_*"
 
 db = lancedb.connect("/tmp/db")
-func = get_registry().get("openai").create(name="text-embedding-ada-002")
+func = get_registry().get("jina").create(name="jina-clip-v1")
 
 class Words(LanceModel):
     text: str = func.SourceField()
@@ -44,31 +132,3 @@ query = "greetings"
 actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```
-
-### Sentence Transformers Embedding function
-LanceDB registers the Sentence Transformers embeddings function in the registry as `sentence-transformers`. You can pass any supported model name to the `create`. By default it uses `"sentence-transformers/paraphrase-MiniLM-L6-v2"`.
-
-```python
-import lancedb
-from lancedb.pydantic import LanceModel, Vector
-from lancedb.embeddings import get_registry
-
-db = lancedb.connect("/tmp/db")
-model = get_registry().get("sentence-transformers").create(name="BAAI/bge-small-en-v1.5", device="cpu")
-
-class Words(LanceModel):
-    text: str = model.SourceField()
-    vector: Vector(model.ndims()) = model.VectorField()
-
-table = db.create_table("words", schema=Words)
-table.add(
-    [
-        {"text": "hello world"},
-        {"text": "goodbye world"}
-    ]
-)
-
-query = "greetings"
-actual = table.search(query).limit(1).to_pydantic(Words)[0]
-print(actual.text)
-```

docs/src/guides/storage.md

@@ -32,28 +32,54 @@ LanceDB OSS supports object stores such as AWS S3 (and compatible stores), Azure
     db = lancedb.connect("az://bucket/path")
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    AWS S3:
+    === "@lancedb/lancedb"
 
-    ```javascript
+        AWS S3:
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("s3://bucket/path");
-    ```
 
-    Google Cloud Storage:
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("s3://bucket/path");
+        ```
 
-    ```javascript
+        Google Cloud Storage:
-    const lancedb = require("lancedb");
-    const db = await lancedb.connect("gs://bucket/path");
-    ```
 
-    Azure Blob Storage:
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("gs://bucket/path");
+        ```
 
-    ```javascript
+        Azure Blob Storage:
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect("az://bucket/path");
+        ```ts
-    ```
+        import * as lancedb from "@lancedb/lancedb";
+        const db = await lancedb.connect("az://bucket/path");
+        ```
+
+
+    === "vectordb (deprecated)"
+
+        AWS S3:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path");
+        ```
+
+        Google Cloud Storage:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("gs://bucket/path");
+        ```
+
+        Azure Blob Storage:
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("az://bucket/path");
+        ```
 
 In most cases, when running in the respective cloud and permissions are set up correctly, no additional configuration is required. When running outside of the respective cloud, authentication credentials must be provided. Credentials and other configuration options can be set in two ways: first, by setting environment variables. And second, by passing a `storage_options` object to the `connect` function. For example, to increase the request timeout to 60 seconds, you can set the `TIMEOUT` environment variable to `60s`:
 
@@ -78,13 +104,26 @@ If you only want this to apply to one particular connection, you can pass the `s
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect("s3://bucket/path",
+        ```ts
-                                     {storageOptions: {timeout: "60s"}});
+        import * as lancedb from "@lancedb/lancedb";
-    ```
+
+        const db = await lancedb.connect("s3://bucket/path", {
+            storageOptions: {timeout: "60s"}
+        });
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path", {
+            storageOptions: {timeout: "60s"}
+        });
+        ```
 
 Getting even more specific, you can set the `timeout` for only a particular table:
 
@@ -101,18 +140,33 @@ Getting even more specific, you can set the `timeout` for only a particular tabl
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    <!-- skip-test -->
+    === "@lancedb/lancedb"
-    ```javascript
+
-    const lancedb = require("lancedb");
+        <!-- skip-test -->
-    const db = await lancedb.connect("s3://bucket/path");
+        ```ts
-    const table = db.createTable(
+        import * as lancedb from "@lancedb/lancedb";
-        "table",
+        const db = await lancedb.connect("s3://bucket/path");
-        [{ a: 1, b: 2}],
+        const table = db.createTable(
-        {storageOptions: {timeout: "60s"}}
+            "table",
-    );
+            [{ a: 1, b: 2}],
-    ```
+            {storageOptions: {timeout: "60s"}}
+        );
+        ```
+
+    === "vectordb (deprecated)"
+
+        <!-- skip-test -->
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect("s3://bucket/path");
+        const table = db.createTable(
+            "table",
+            [{ a: 1, b: 2}],
+            {storageOptions: {timeout: "60s"}}
+        );
+        ```
 
 !!! info "Storage option casing"
 
@@ -135,7 +189,6 @@ There are several options that can be set for all object stores, mostly related
 | `proxy_ca_certificate`     | PEM-formatted CA certificate for proxy connections.                                                |
 | `proxy_excludes`           | List of hosts that bypass the proxy. This is a comma-separated list of domains and IP masks. Any subdomain of the provided domain will be bypassed. For example, `example.com, 192.168.1.0/24` would bypass `https://api.example.com`, `https://www.example.com`, and any IP in the range `192.168.1.0/24`. |
 
-
 ### AWS S3
 
 To configure credentials for AWS S3, you can use the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` keys. Region can also be set, but it is not mandatory when using AWS.
@@ -155,21 +208,39 @@ These can be set as environment variables or passed in the `storage_options` par
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "s3://bucket/path",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "s3://bucket/path",
-                awsAccessKeyId: "my-access-key",
+            {
-                awsSecretAccessKey: "my-secret-key",
+                storageOptions: {
-                awsSessionToken: "my-session-token",
+                    awsAccessKeyId: "my-access-key",
+                    awsSecretAccessKey: "my-secret-key",
+                    awsSessionToken: "my-session-token",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    awsAccessKeyId: "my-access-key",
+                    awsSecretAccessKey: "my-secret-key",
+                    awsSessionToken: "my-session-token",
+                }
+            }
+        );
+        ```
 
 Alternatively, if you are using AWS SSO, you can use the `AWS_PROFILE` and `AWS_DEFAULT_REGION` environment variables.
 
@@ -188,7 +259,6 @@ The following keys can be used as both environment variables or keys in the `sto
 | `aws_sse_kms_key_id`                | The KMS key ID to use for server-side encryption. If set, `aws_server_side_encryption` must be `"aws:kms"` or `"aws:kms:dsse"`. |
 | `aws_sse_bucket_key_enabled`        | Whether to use bucket keys for server-side encryption.                                               |
 
-
 !!! tip "Automatic cleanup for failed writes"
 
     LanceDB uses [multi-part uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html) when writing data to S3 in order to maximize write speed. LanceDB will abort these uploads when it shuts down gracefully, such as when cancelled by keyboard interrupt. However, in the rare case that LanceDB crashes, it is possible that some data will be left lingering in your account. To cleanup this data, we recommend (as AWS themselves do) that you setup a lifecycle rule to delete in-progress uploads after 7 days. See the AWS guide:
@@ -265,6 +335,108 @@ For **read-only access**, LanceDB will need a policy such as:
 }
 ```
 
+#### DynamoDB Commit Store for concurrent writes
+
+By default, S3 does not support concurrent writes. Having two or more processes
+writing to the same table at the same time can lead to data corruption. This is
+because S3, unlike other object stores, does not have any atomic put or copy
+operation.
+
+To enable concurrent writes, you can configure LanceDB to use a DynamoDB table
+as a commit store. This table will be used to coordinate writes between
+different processes. To enable this feature, you must modify your connection
+URI to use the `s3+ddb` scheme and add a query parameter `ddbTableName` with the
+name of the table to use.
+
+=== "Python"
+
+    ```python
+    import lancedb
+    db = await lancedb.connect_async(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    )
+    ```
+
+=== "JavaScript"
+
+    ```javascript
+    const lancedb = require("lancedb");
+
+    const db = await lancedb.connect(
+        "s3+ddb://bucket/path?ddbTableName=my-dynamodb-table",
+    );
+    ```
+
+The DynamoDB table must be created with the following schema:
+
+- Hash key: `base_uri` (string)
+- Range key: `version` (number)
+
+You can create this programmatically with:
+
+=== "Python"
+
+    <!-- skip-test -->
+    ```python
+    import boto3
+
+    dynamodb = boto3.client("dynamodb")
+    table = dynamodb.create_table(
+        TableName=table_name,
+        KeySchema=[
+            {"AttributeName": "base_uri", "KeyType": "HASH"},
+            {"AttributeName": "version", "KeyType": "RANGE"},
+        ],
+        AttributeDefinitions=[
+            {"AttributeName": "base_uri", "AttributeType": "S"},
+            {"AttributeName": "version", "AttributeType": "N"},
+        ],
+        ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
+    )
+    ```
+
+=== "JavaScript"
+
+    <!-- skip-test -->
+    ```javascript
+    import {
+      CreateTableCommand,
+      DynamoDBClient,
+    } from "@aws-sdk/client-dynamodb";
+
+    const dynamodb = new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+    const command = new CreateTableCommand({
+      TableName: table_name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    ```
+
+
 #### S3-compatible stores
 
 LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you must specify both region and endpoint:
@@ -282,20 +454,37 @@ LanceDB can also connect to S3-compatible stores, such as MinIO. To do so, you m
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "s3://bucket/path",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "s3://bucket/path",
-                region: "us-east-1",
+            {
-                endpoint: "http://minio:9000",
+                storageOptions: {
+                    region: "us-east-1",
+                    endpoint: "http://minio:9000",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://bucket/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    endpoint: "http://minio:9000",
+                }
+            }
+        );
+        ```
 
 This can also be done with the ``AWS_ENDPOINT`` and ``AWS_DEFAULT_REGION`` environment variables.
 
@@ -326,21 +515,37 @@ To configure LanceDB to use an S3 Express endpoint, you must set the storage opt
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "s3://my-bucket--use1-az4--x-s3/path",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "s3://my-bucket--use1-az4--x-s3/path",
-                region: "us-east-1",
+            {
-                s3Express: "true",
+                storageOptions: {
+                    region: "us-east-1",
+                    s3Express: "true",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
 
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "s3://my-bucket--use1-az4--x-s3/path",
+            {
+                storageOptions: {
+                    region: "us-east-1",
+                    s3Express: "true",
+                }
+            }
+        );
+        ```
 
 ### Google Cloud Storage
 
@@ -359,26 +564,40 @@ GCS credentials are configured by setting the `GOOGLE_SERVICE_ACCOUNT` environme
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "gs://my-bucket/my-database",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "gs://my-bucket/my-database",
-                serviceAccount: "path/to/service-account.json",
+            {
+                storageOptions: {
+                    serviceAccount: "path/to/service-account.json",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
 
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "gs://my-bucket/my-database",
+            {
+                storageOptions: {
+                    serviceAccount: "path/to/service-account.json",
+                }
+            }
+        );
+        ```
 
 !!! info "HTTP/2 support"
 
     By default, GCS uses HTTP/1 for communication, as opposed to HTTP/2. This improves maximum throughput significantly. However, if you wish to use HTTP/2 for some reason, you can set the environment variable `HTTP1_ONLY` to `false`.
 
-
 The following keys can be used as both environment variables or keys in the `storage_options` parameter:
 <!-- source: https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html -->
 
@@ -388,7 +607,6 @@ The following keys can be used as both environment variables or keys in the `sto
 | ``google_service_account_key``        | The serialized service account key.          |
 | ``google_application_credentials``    | Path to the application credentials.         |
 
-
 ### Azure Blob Storage
 
 Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_ACCOUNT_NAME`and `AZURE_STORAGE_ACCOUNT_KEY` environment variables. Alternatively, you can pass the account name and key in the `storage_options` parameter:
@@ -407,20 +625,37 @@ Azure Blob Storage credentials can be configured by setting the `AZURE_STORAGE_A
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    const lancedb = require("lancedb");
+
-    const db = await lancedb.connect(
+        ```ts
-        "az://my-container/my-database",
+        import * as lancedb from "@lancedb/lancedb";
-        {
+        const db = await lancedb.connect(
-            storageOptions: {
+            "az://my-container/my-database",
-                accountName: "some-account",
+            {
-                accountKey: "some-key",
+                storageOptions: {
+                    accountName: "some-account",
+                    accountKey: "some-key",
+                }
             }
-        }
+        );
-    );
+        ```
-    ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        const lancedb = require("lancedb");
+        const db = await lancedb.connect(
+            "az://my-container/my-database",
+            {
+                storageOptions: {
+                    accountName: "some-account",
+                    accountKey: "some-key",
+                }
+            }
+        );
+        ```
 
 These keys can be used as both environment variables or keys in the `storage_options` parameter:
 

docs/src/guides/tables.md

@@ -8,27 +8,41 @@ This guide will show how to create tables, insert data into them, and update the
 
 ## Creating a LanceDB Table
 
+Initialize a LanceDB connection and create a table
+
 === "Python"
-    Initialize a LanceDB connection and create a table using one of the many methods listed below.
 
     ```python
     import lancedb
     db = lancedb.connect("./.lancedb")
     ```
 
-=== "Javascript"
-
-    Initialize a VectorDB connection and create a table using one of the many methods listed below.
-
-    ```javascript
-    const lancedb = require("vectordb");
-
-    const uri = "data/sample-lancedb";
-    const db = await lancedb.connect(uri);
-    ```
-
     LanceDB allows ingesting data from various sources - `dict`, `list[dict]`, `pd.DataFrame`, `pa.Table` or a `Iterator[pa.RecordBatch]`. Let's take a look at some of the these.
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        import * as lancedb from "@lancedb/lancedb";
+        import * as arrow from "apache-arrow";
+
+        const uri = "data/sample-lancedb";
+        const db = await lancedb.connect(uri);
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        const lancedb = require("vectordb");
+        const arrow = require("apache-arrow");
+
+        const uri = "data/sample-lancedb";
+        const db = await lancedb.connect(uri);
+        ```
+
+
+
 ### From list of tuples or dictionaries
 
 === "Python"
@@ -45,6 +59,7 @@ This guide will show how to create tables, insert data into them, and update the
 
     db["my_table"].head()
     ```
+
     !!! info "Note"
         If the table already exists, LanceDB will raise an error by default.
 
@@ -52,90 +67,137 @@ This guide will show how to create tables, insert data into them, and update the
         and the table exists, then it simply opens the existing table. The data you
         passed in will NOT be appended to the table in that case.
 
-        ```python
+    ```python
-        db.create_table("name", data, exist_ok=True)
+    db.create_table("name", data, exist_ok=True)
+    ```
+
+    Sometimes you want to make sure that you start fresh. If you want to
+    overwrite the table, you can pass in mode="overwrite" to the createTable function.
+
+    ```python
+    db.create_table("name", data, mode="overwrite")
+    ```
+
+=== "Typescript[^1]"
+    You can create a LanceDB table in JavaScript using an array of records as follows.
+
+    === "@lancedb/lancedb"
+
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table"
+        ```
+
+        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use `apache-arrow` to declare a schema
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_with_schema"
+        ```
+
+        !!! info "Note"
+            `createTable` supports an optional `existsOk` parameter. When set to true
+            and the table exists, then it simply opens the existing table. The data you
+            passed in will NOT be appended to the table in that case.
+
+        ```ts
+        --8<-- "nodejs/examples/basic.ts:create_table_exists_ok"
         ```
 
         Sometimes you want to make sure that you start fresh. If you want to
-        overwrite the table, you can pass in mode="overwrite" to the createTable function.
+        overwrite the table, you can pass in mode: "overwrite" to the createTable function.
 
-        ```python
+        ```ts
-        db.create_table("name", data, mode="overwrite")
+        --8<-- "nodejs/examples/basic.ts:create_table_overwrite"
         ```
 
-=== "Javascript"
+    === "vectordb (deprecated)"
-    You can create a LanceDB table in JavaScript using an array of JSON records as follows.
 
-    ```javascript
+        ```ts
-    const tb = await db.createTable("my_table", [{
+        --8<-- "docs/src/basic_legacy.ts:create_table"
-        "vector": [3.1, 4.1],
+        ```
-        "item": "foo",
-        "price": 10.0
-    }, {
-        "vector": [5.9, 26.5],
-        "item": "bar",
-        "price": 20.0
-    }]);
-    ```
-    !!! info "Note"
-    If the table already exists, LanceDB will raise an error by default. If you want to overwrite the table, you need to specify the `WriteMode` in the createTable function.
 
-    ```javascript
+        This will infer the schema from the provided data. If you want to explicitly provide a schema, you can use apache-arrow to declare a schema
-    const table = await con.createTable(tableName, data, { writeMode: WriteMode.Overwrite })
-    ```
 
-    ### From a Pandas DataFrame
 
-    ```python
-    import pandas as pd
 
-    data = pd.DataFrame({
+        ```ts
-        "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+        --8<-- "docs/src/basic_legacy.ts:create_table_with_schema"
-        "lat": [45.5, 40.1],
+        ```
-        "long": [-122.7, -74.1]
-    })
 
-    db.create_table("my_table", data)
+        !!! warning
+            `existsOk` is not available in `vectordb`
 
-    db["my_table"].head()
+
-    ```
+
-    !!! info "Note"
+            If the table already exists, vectordb will raise an error by default.
+            You can use `writeMode: WriteMode.Overwrite` to overwrite the table.
+            But this will delete the existing table and create a new one with the same name.
+
+
+        Sometimes you want to make sure that you start fresh.
+
+        If you want to overwrite the table, you can pass in `writeMode: lancedb.WriteMode.Overwrite` to the createTable function.
+
+        ```ts
+        const table = await con.createTable(tableName, data, {
+            writeMode: WriteMode.Overwrite
+        })
+        ```
+
+### From a Pandas DataFrame
+
+```python
+import pandas as pd
+
+data = pd.DataFrame({
+    "vector": [[1.1, 1.2, 1.3, 1.4], [0.2, 1.8, 0.4, 3.6]],
+    "lat": [45.5, 40.1],
+    "long": [-122.7, -74.1]
+})
+
+db.create_table("my_table", data)
+
+db["my_table"].head()
+```
+
+!!! info "Note"
     Data is converted to Arrow before being written to disk. For maximum control over how data is saved, either provide the PyArrow schema to convert to or else provide a PyArrow Table directly.
 
-    The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type. 
+The **`vector`** column needs to be a [Vector](../python/pydantic.md#vector-field) (defined as [pyarrow.FixedSizeList](https://arrow.apache.org/docs/python/generated/pyarrow.list_.html)) type.
 
-    ```python
+```python
-    custom_schema = pa.schema([
+custom_schema = pa.schema([
-    pa.field("vector", pa.list_(pa.float32(), 4)),
+pa.field("vector", pa.list_(pa.float32(), 4)),
-    pa.field("lat", pa.float32()),
+pa.field("lat", pa.float32()),
-    pa.field("long", pa.float32())
+pa.field("long", pa.float32())
-    ])
+])
 
-    table = db.create_table("my_table", data, schema=custom_schema)
+table = db.create_table("my_table", data, schema=custom_schema)
-    ```
+```
 
 ### From a Polars DataFrame
 
-    LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
+LanceDB supports [Polars](https://pola.rs/), a modern, fast DataFrame library
-    written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
+written in Rust. Just like in Pandas, the Polars integration is enabled by PyArrow
-    under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
+under the hood. A deeper integration between LanceDB Tables and Polars DataFrames
-    is on the way.
+is on the way.
 
-    ```python
+```python
-    import polars as pl
+import polars as pl
 
-    data = pl.DataFrame({
+data = pl.DataFrame({
-        "vector": [[3.1, 4.1], [5.9, 26.5]],
+    "vector": [[3.1, 4.1], [5.9, 26.5]],
-        "item": ["foo", "bar"],
+    "item": ["foo", "bar"],
-        "price": [10.0, 20.0]
+    "price": [10.0, 20.0]
-    })
+})
-    table = db.create_table("pl_table", data=data)
+table = db.create_table("pl_table", data=data)
-    ```
+```
 
 ### From an Arrow Table
+You can also create LanceDB tables directly from Arrow tables.
+LanceDB supports float16 data type!
+
 === "Python"
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports float16 data type!
 
     ```python
     import pyarrows as pa
@@ -160,13 +222,19 @@ This guide will show how to create tables, insert data into them, and update the
     tbl = db.create_table("f16_tbl", data, schema=schema)
     ```
 
-=== "Javascript"
+=== "Typescript[^1]"
-    You can also create LanceDB tables directly from Arrow tables. 
-    LanceDB supports Float16 data type!
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/basic_legacy.ts:create_f16_table"
+
-    ```
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_f16_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_f16_table"
+        ```
 
 ### From Pydantic Models
 
@@ -329,23 +397,24 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
     tbl = db.open_table("my_table")
     ```
 
-=== "JavaScript"
+=== "Typescript[^1]"
+
     If you forget the name of your table, you can always get a listing of all table names.
 
-    ```javascript
+    ```typescript
     console.log(await db.tableNames());
     ```
 
     Then, you can open any existing tables.
 
-    ```javascript
+    ```typescript
     const tbl = await db.openTable("my_table");
     ```
 
 ## Creating empty table
+You can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
 
 === "Python"
-    In Python, you can create an empty table for scenarios where you want to add data to the table later. An example would be when you want to collect data from a stream/external file and then add it to a table in batches.
 
     ```python
 
@@ -382,9 +451,23 @@ You can also use iterators of other types like Pandas DataFrame or Pylists direc
 
     Once the empty table has been created, you can add data to it via the various methods listed in the [Adding to a table](#adding-to-a-table) section.
 
+=== "Typescript[^1]"
+
+    === "@lancedb/lancedb"
+
+        ```typescript
+        --8<-- "nodejs/examples/basic.ts:create_empty_table"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```typescript
+        --8<-- "docs/src/basic_legacy.ts:create_empty_table"
+        ```
+
 ## Adding to a table
 
-After a table has been created, you can always add more data to it using the various methods available.
+After a table has been created, you can always add more data to it usind the `add` method
 
 === "Python"
     You can add any of the valid data structures accepted by LanceDB table, i.e, `dict`, `list[dict]`, `pd.DataFrame`, or `Iterator[pa.RecordBatch]`. Below are some examples.
@@ -472,9 +555,7 @@ After a table has been created, you can always add more data to it using the var
         tbl.add(models)
         ```
 
-
+=== "Typescript[^1]"
-
-=== "JavaScript"
 
     ```javascript
     await tbl.add(
@@ -530,15 +611,15 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
     # 0  3  [5.0, 6.0]
     ```
 
-=== "JavaScript"
+=== "Typescript[^1]"
 
-    ```javascript
+    ```ts
     await tbl.delete('item = "fizz"')
     ```
 
     ### Deleting row with specific column value
 
-    ```javascript
+    ```ts
     const con = await lancedb.connect("./.lancedb")
     const data = [
       {id: 1, vector: [1, 2]},
@@ -552,7 +633,7 @@ Use the `delete()` method on tables to delete rows from a table. To choose which
 
     ### Delete from a list of values
 
-    ```javascript
+    ```ts
     const to_remove = [1, 5];
     await tbl.delete(`id IN (${to_remove.join(",")})`)
     await tbl.countRows() // Returns 1
@@ -609,26 +690,49 @@ This can be used to update zero to all rows depending on how many rows match the
     2  2  [10.0, 10.0]
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
-    API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
+    === "@lancedb/lancedb"
 
-    ```javascript
+        API Reference: [lancedb.Table.update](../js/classes/Table.md/#update)
-    const lancedb = require("vectordb");
 
-    const db = await lancedb.connect("./.lancedb");
+        ```ts
+        import * as lancedb from "@lancedb/lancedb";
 
-    const data = [
+        const db = await lancedb.connect("./.lancedb");
-      {x: 1, vector: [1, 2]},
-      {x: 2, vector: [3, 4]},
-      {x: 3, vector: [5, 6]},
-    ];
-    const tbl = await db.createTable("my_table", data)
 
-    await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        const data = [
-    ```
+            {x: 1, vector: [1, 2]},
+            {x: 2, vector: [3, 4]},
+            {x: 3, vector: [5, 6]},
+        ];
+        const tbl = await db.createTable("my_table", data)
 
-The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
+        await tbl.update({vector: [10, 10]}, { where: "x = 2"})
+        ```
+
+    === "vectordb (deprecated)"
+
+        API Reference: [vectordb.Table.update](../javascript/interfaces/Table.md/#update)
+
+        ```ts
+        const lancedb = require("vectordb");
+
+        const db = await lancedb.connect("./.lancedb");
+
+        const data = [
+            {x: 1, vector: [1, 2]},
+            {x: 2, vector: [3, 4]},
+            {x: 3, vector: [5, 6]},
+        ];
+        const tbl = await db.createTable("my_table", data)
+
+        await tbl.update({ where: "x = 2", values: {vector: [10, 10]} })
+        ```
+
+#### Updating using a sql query
+
+  The `values` parameter is used to provide the new values for the columns as literal values. You can also use the `values_sql` / `valuesSql` parameter to provide SQL expressions for the new values. For example, you can use `values_sql="x + 1"` to increment the value of the `x` column by 1.
 
 === "Python"
 
@@ -647,11 +751,17 @@ The `values` parameter is used to provide the new values for the columns as lite
     2  3  [10.0, 10.0]
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    await tbl.update({ valuesSql: { x: "x + 1" } })
+
-    ```
+        Coming Soon!
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        await tbl.update({ valuesSql: { x: "x + 1" } })
+        ```
 
 !!! info "Note"
 
@@ -672,7 +782,7 @@ Use the `drop_table()` method on the database to remove a table.
       By default, if the table does not exist an exception is raised. To suppress this,
       you can pass in `ignore_missing=True`.
 
-=== "Javascript/Typescript"
+=== "TypeScript"
 
       ```typescript
       --8<-- "docs/src/basic_legacy.ts:drop_table"
@@ -726,18 +836,18 @@ There are three possible settings for `read_consistency_interval`:
     table.checkout_latest()
     ```
 
-=== "JavaScript/Typescript"
+=== "Typescript[^1]"
 
     To set strong consistency, use `0`:
 
-    ```javascript
+    ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 0 });
     const table = await db.openTable("my_table");
     ```
 
     For eventual consistency, specify the update interval as seconds:
 
-    ```javascript
+    ```ts
     const db = await lancedb.connect({ uri: "./.lancedb", readConsistencyInterval: 5 });
     const table = await db.openTable("my_table");
     ```
@@ -749,3 +859,5 @@ There are three possible settings for `read_consistency_interval`:
 ## What's next?
 
 Learn the best practices on creating an ANN index and getting the most out of it.
+
+[^1]: The `vectordb` package is a legacy package that is  deprecated in favor of `@lancedb/lancedb`.  The `vectordb` package will continue to receive bug fixes and security updates until September 2024.  We recommend all new projects use `@lancedb/lancedb`.  See the [migration guide](migration.md) for more information.

docs/src/guides/tuning_retrievers/1_query_types.md

@@ -1,4 +1,7 @@
 ## Improving retriever performance
+
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
 VectorDBs are used as retreivers in recommender or chatbot-based systems for retrieving relevant data based on user queries. For example, retriever is a critical component of Retrieval Augmented Generation (RAG) acrhitectures. In this section, we will discuss how to improve the performance of retrievers.
 
 There are serveral ways to improve the performance of retrievers. Some of the common techniques are:

docs/src/guides/tuning_retrievers/2_reranking.md

@@ -1,4 +1,6 @@
-Continuing from the previous example, we can now rerank the results using more complex rerankers.
+Continuing from the previous section, we can now rerank the results using more complex rerankers.
+
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/lancedb_reranking.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
 
 ## Reranking search results
 You can rerank any search results using a reranker. The syntax for reranking is as follows:

docs/src/guides/tuning_retrievers/3_embed_tuning.md

@@ -0,0 +1,82 @@
+## Finetuning the Embedding Model
+Try it yourself - <a href="https://colab.research.google.com/github/lancedb/lancedb/blob/main/docs/src/notebooks/embedding_tuner.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"></a><br/>
+
+Another way to improve retriever performance is to fine-tune the embedding model itself. Fine-tuning the embedding model can help in learning better representations for the documents and queries in the dataset. This can be particularly useful when the dataset is very different from the pre-trained data used to train the embedding model.
+
+We'll use the same dataset as in the previous sections. Start off by splitting the dataset into training and validation sets:
+```python
+from sklearn.model_selection import train_test_split
+
+train_df, validation_df = train_test_split("data_qa.csv", test_size=0.2, random_state=42)
+
+train_df.to_csv("data_train.csv", index=False)
+validation_df.to_csv("data_val.csv", index=False)
+```
+
+You can use any tuning API to fine-tune embedding models. In this example, we'll utilise Llama-index as it also comes with utilities for synthetic data generation and training the model. 
+
+
+Then parse the dataset as llama-index text nodes and generate synthetic QA pairs from each node.
+```python
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.readers.file import PagedCSVReader
+from llama_index.finetuning import generate_qa_embedding_pairs
+from llama_index.core.evaluation import EmbeddingQAFinetuneDataset
+
+def load_corpus(file):
+    loader = PagedCSVReader(encoding="utf-8")
+    docs = loader.load_data(file=Path(file))
+
+    parser = SentenceSplitter()
+    nodes = parser.get_nodes_from_documents(docs)
+
+    return nodes
+
+from llama_index.llms.openai import OpenAI
+
+
+train_dataset = generate_qa_embedding_pairs(
+    llm=OpenAI(model="gpt-3.5-turbo"), nodes=train_nodes, verbose=False
+)
+val_dataset = generate_qa_embedding_pairs(
+    llm=OpenAI(model="gpt-3.5-turbo"), nodes=val_nodes, verbose=False
+)
+```
+
+Now we'll use `SentenceTransformersFinetuneEngine` engine to fine-tune the model. You can also use `sentence-transformers` or `transformers` library to fine-tune the model. 
+
+```python
+from llama_index.finetuning import SentenceTransformersFinetuneEngine
+
+finetune_engine = SentenceTransformersFinetuneEngine(
+    train_dataset,
+    model_id="BAAI/bge-small-en-v1.5",
+    model_output_path="tuned_model",
+    val_dataset=val_dataset,
+)
+finetune_engine.finetune()
+embed_model = finetune_engine.get_finetuned_model()
+```
+This saves the fine tuned embedding model in `tuned_model` folder. This al
+
+# Evaluation results
+In order to eval the retriever, you can either use this model to ingest the data into LanceDB directly or llama-index's LanceDB integration to create a `VectorStoreIndex` and use it as a retriever. 
+On performing the same hit-rate evaluation as before, we see a significant improvement in the hit-rate across all query types.
+
+### Baseline
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.640 |
+| Full-text Search | 0.595 |
+| Reranked Vector Search | 0.677 |
+| Reranked Full-text Search | 0.672 |
+| Hybrid Search (w/ CohereReranker) | 0.759|
+
+### Fine-tuned model ( 2 iterations )
+| Query Type | Hit-rate@5 |
+| --- | --- |
+| Vector Search | 0.672 |
+| Full-text Search | 0.595 |
+| Reranked Vector Search | 0.754 |
+| Reranked Full-text Search | 0.672|
+| Hybrid Search (w/ CohereReranker) | 0.768 |

docs/src/integrations/langchain.md

@@ -2,7 +2,7 @@
 ![Illustration](../assets/langchain.png)
 
 ## Quick Start
-You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model.
+You can load your document data using langchain's loaders, for this example we are using `TextLoader` and `OpenAIEmbeddings` as the embedding model. Checkout Complete example here - [LangChain demo](../notebooks/langchain_example.ipynb)
 ```python
 import os
 from langchain.document_loaders import TextLoader
@@ -38,6 +38,8 @@ The exhaustive list of parameters for `LanceDB` vector store are :
 - `api_key`: (Optional) API key to use for LanceDB cloud database. Defaults to `None`.  
 - `region`: (Optional) Region to use for LanceDB cloud database. Only for LanceDB Cloud, defaults to `None`.  
 - `mode`: (Optional) Mode to use for adding data to the table. Defaults to `'overwrite'`.  
+- `reranker`: (Optional) The reranker to use for LanceDB.
+- `relevance_score_fn`: (Optional[Callable[[float], float]]) Langchain relevance score function to be used. Defaults to `None`. 
 
 ```python
 db_url = "db://lang_test" # url of db you created
@@ -54,12 +56,14 @@ vector_store = LanceDB(
 ```
 
 ### Methods 
-To add texts and store respective embeddings automatically:   
+
 ##### add_texts()
 - `texts`: `Iterable` of strings to add to the vectorstore.
 - `metadatas`: Optional `list[dict()]` of metadatas associated with the texts.
 - `ids`: Optional `list` of ids to associate with the texts. 
+- `kwargs`: `Any`
 
+This method adds texts and stores respective embeddings automatically.
 
 ```python
 vector_store.add_texts(texts = ['test_123'], metadatas =[{'source' :'wiki'}]) 
@@ -74,7 +78,6 @@ pd_df.to_csv("docsearch.csv", index=False)
 # you can also create a new vector store object using an older connection object:
 vector_store = LanceDB(connection=tbl, embedding=embeddings)
 ```
-For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
 ##### create_index() 
 - `col_name`: `Optional[str] = None`
 - `vector_col`: `Optional[str] = None`
@@ -82,6 +85,8 @@ For index creation make sure your table has enough data in it. An ANN index is u
 - `num_sub_vectors`: `Optional[int] = 96`
 - `index_cache_size`: `Optional[int] = None`
 
+This method creates an index for the vector store. For index creation make sure your table has enough data in it. An ANN index is ususally not needed for datasets ~100K vectors. For large-scale (>1M) or higher dimension vectors, it is beneficial to create an ANN index.
+
 ```python
 # for creating vector index
 vector_store.create_index(vector_col='vector', metric = 'cosine')
@@ -90,3 +95,107 @@ vector_store.create_index(vector_col='vector', metric = 'cosine')
 vector_store.create_index(col_name='text')
 
 ```
+
+##### similarity_search()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `fts`: `Optional[bool] = False`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Return documents most similar to the query without relevance scores
+
+```python
+docs = docsearch.similarity_search(query)
+print(docs[0].page_content)
+```
+
+##### similarity_search_by_vector()
+- `embedding`: `List[float]`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Returns documents most similar to the query vector.
+
+```python
+docs = docsearch.similarity_search_by_vector(query)
+print(docs[0].page_content)
+```
+
+##### similarity_search_with_score()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `kwargs`: `Any`
+
+Returns documents most similar to the query string with relevance scores, gets called by base class's `similarity_search_with_relevance_scores` which selects relevance score based on our `_select_relevance_score_fn`.
+
+```python
+docs = docsearch.similarity_search_with_relevance_scores(query)
+print("relevance score - ", docs[0][1])
+print("text- ", docs[0][0].page_content[:1000])
+```
+
+##### similarity_search_by_vector_with_relevance_scores()
+- `embedding`: `List[float]`
+- `k`: `Optional[int] = None`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `name`: `Optional[str] = None`
+- `kwargs`: `Any`
+
+Return documents most similar to the query vector with relevance scores.
+Relevance score 
+
+```python
+docs = docsearch.similarity_search_by_vector_with_relevance_scores(query_embedding)
+print("relevance score - ", docs[0][1])
+print("text- ", docs[0][0].page_content[:1000])
+```
+
+##### max_marginal_relevance_search()
+- `query`: `str`
+- `k`: `Optional[int] = None`
+- `fetch_k` : Number of Documents to fetch to pass to MMR algorithm, `Optional[int] = None`
+- `lambda_mult`: Number between 0 and 1 that determines the degree
+                        of diversity among the results with 0 corresponding
+                        to maximum diversity and 1 to minimum diversity.
+                        Defaults to 0.5. `float = 0.5`
+- `filter`: `Optional[Dict[str, str]] = None`
+- `kwargs`: `Any`
+
+Returns docs selected using the maximal marginal relevance(MMR).
+Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents.
+
+Similarly, `max_marginal_relevance_search_by_vector()` function returns docs most similar to the embedding passed to the function using MMR. instead of a string query you need to pass the embedding to be searched for. 
+
+```python
+result = docsearch.max_marginal_relevance_search(
+        query="text"
+    )
+result_texts = [doc.page_content for doc in result]
+print(result_texts)
+
+## search by vector :
+result = docsearch.max_marginal_relevance_search_by_vector(
+        embeddings.embed_query("text")
+    )
+result_texts = [doc.page_content for doc in result]
+print(result_texts)
+```
+
+##### add_images()
+- `uris` : File path to the image. `List[str]`.
+- `metadatas` : Optional list of metadatas. `(Optional[List[dict]], optional)`
+- `ids` : Optional list of IDs. `(Optional[List[str]], optional)`
+
+Adds images by automatically creating their embeddings and adds them to the vectorstore.
+
+```python
+vec_store.add_images(uris=image_uris) 
+# here image_uris are local fs paths to the images.
+```
+
+

docs/src/integrations/llamaIndex.md

@@ -2,7 +2,8 @@
 ![Illustration](../assets/llama-index.jpg)
 
 ## Quick start
-You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. You can run the below script to try it out :
+You would need to install the integration via `pip install llama-index-vector-stores-lancedb` in order to use it. 
+You can run the below script to try it out :
 ```python
 import logging
 import sys
@@ -43,6 +44,8 @@ retriever = index.as_retriever(vector_store_kwargs={"where": lance_filter})
 response = retriever.retrieve("What did the author do growing up?")
 ```
 
+Checkout Complete example here - [LlamaIndex demo](../notebooks/LlamaIndex_example.ipynb)
+
 ### Filtering
 For metadata filtering, you can use a Lance SQL-like string filter as demonstrated in the example above. Additionally, you can also filter using the `MetadataFilters` class from LlamaIndex:
 ```python

docs/src/js/README.md

@@ -1,4 +1,6 @@
-@lancedb/lancedb / [Exports](modules.md)
+**@lancedb/lancedb** • [**Docs**](globals.md)
+
+***
 
 # LanceDB JavaScript SDK
 
@@ -45,29 +47,20 @@ npm run test
 
 ### Running lint / format
 
-LanceDb uses eslint for linting. VSCode does not need any plugins to use eslint. However, it
+LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
-may need some additional configuration. Make sure that eslint.experimental.useFlatConfig is
+To manually lint your code you can run:
-set to true. Also, if your vscode root folder is the repo root then you will need to set
-the eslint.workingDirectories to ["nodejs"]. To manually lint your code you can run:
 
 ```sh
 npm run lint
 ```
 
-LanceDb uses prettier for formatting. If you are using VSCode you will need to install the
+to automatically fix all fixable issues:
-"Prettier - Code formatter" extension. You should then configure it to be the default formatter
-for typescript and you should enable format on save. To manually check your code's format you
-can run:
 
 ```sh
-npm run chkformat
+npm run lint-fix
 ```
 
-If you need to manually format your code you can run:
+If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
-
-```sh
-npx prettier --write .
-```
 
 ### Generating docs
 

docs/src/js/classes/Connection.md

@@ -1,6 +1,10 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Connection
+[**@lancedb/lancedb**](../README.md) • **Docs**
 
-# Class: Connection
+***
+
+[@lancedb/lancedb](../globals.md) / Connection
+
+# Class: `abstract` Connection
 
 A LanceDB Connection that allows you to open tables and create new ones.
 
@@ -19,62 +23,21 @@ be closed when they are garbage collected.
 Any created tables are independent and will continue to work even if
 the underlying connection has been closed.
 
-## Table of contents
-
-### Constructors
-
-- [constructor](Connection.md#constructor)
-
-### Properties
-
-- [inner](Connection.md#inner)
-
-### Methods
-
-- [close](Connection.md#close)
-- [createEmptyTable](Connection.md#createemptytable)
-- [createTable](Connection.md#createtable)
-- [display](Connection.md#display)
-- [dropTable](Connection.md#droptable)
-- [isOpen](Connection.md#isopen)
-- [openTable](Connection.md#opentable)
-- [tableNames](Connection.md#tablenames)
-
 ## Constructors
 
-### constructor
+### new Connection()
 
-• **new Connection**(`inner`): [`Connection`](Connection.md)
+> **new Connection**(): [`Connection`](Connection.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `Connection` |
 
 #### Returns
 
 [`Connection`](Connection.md)
 
-#### Defined in
-
-[connection.ts:72](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L72)
-
-## Properties
-
-### inner
-
-• `Readonly` **inner**: `Connection`
-
-#### Defined in
-
-[connection.ts:70](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L70)
-
 ## Methods
 
-### close
+### close()
 
-▸ **close**(): `void`
+> `abstract` **close**(): `void`
 
 Close the connection, releasing any underlying resources.
 
@@ -86,63 +49,78 @@ Any attempt to use the connection after it is closed will result in an error.
 
 `void`
 
-#### Defined in
+***
 
-[connection.ts:88](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L88)
+### createEmptyTable()
 
-___
+> `abstract` **createEmptyTable**(`name`, `schema`, `options`?): `Promise`<[`Table`](Table.md)>
-
-### createEmptyTable
-
-▸ **createEmptyTable**(`name`, `schema`, `options?`): `Promise`\<[`Table`](Table.md)\>
 
 Creates a new empty Table
 
 #### Parameters
 
-| Name | Type | Description |
+• **name**: `string`
-| :------ | :------ | :------ |
+
-| `name` | `string` | The name of the table. |
+The name of the table.
-| `schema` | `Schema`\<`any`\> | The schema of the table |
+
-| `options?` | `Partial`\<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)\> | - |
+• **schema**: `SchemaLike`
+
+The schema of the table
+
+• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
 
 #### Returns
 
-`Promise`\<[`Table`](Table.md)\>
+`Promise`&lt;[`Table`](Table.md)&gt;
 
-#### Defined in
+***
 
-[connection.ts:151](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L151)
+### createTable()
 
-___
+#### createTable(options)
 
-### createTable
+> `abstract` **createTable**(`options`): `Promise`&lt;[`Table`](Table.md)&gt;
-
-▸ **createTable**(`name`, `data`, `options?`): `Promise`\<[`Table`](Table.md)\>
 
 Creates a new Table and initialize it with new data.
 
-#### Parameters
+##### Parameters
 
-| Name | Type | Description |
+• **options**: `object` & `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
-| :------ | :------ | :------ |
-| `name` | `string` | The name of the table. |
-| `data` | `Table`\<`any`\> \| `Record`\<`string`, `unknown`\>[] | Non-empty Array of Records to be inserted into the table |
-| `options?` | `Partial`\<[`CreateTableOptions`](../interfaces/CreateTableOptions.md)\> | - |
 
-#### Returns
+The options object.
 
-`Promise`\<[`Table`](Table.md)\>
+##### Returns
 
-#### Defined in
+`Promise`&lt;[`Table`](Table.md)&gt;
 
-[connection.ts:123](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L123)
+#### createTable(name, data, options)
 
-___
+> `abstract` **createTable**(`name`, `data`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;
 
-### display
+Creates a new Table and initialize it with new data.
 
-▸ **display**(): `string`
+##### Parameters
+
+• **name**: `string`
+
+The name of the table.
+
+• **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
+
+Non-empty Array of Records
+to be inserted into the table
+
+• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+
+##### Returns
+
+`Promise`&lt;[`Table`](Table.md)&gt;
+
+***
+
+### display()
+
+> `abstract` **display**(): `string`
 
 Return a brief description of the connection
 
@@ -150,37 +128,29 @@ Return a brief description of the connection
 
 `string`
 
-#### Defined in
+***
 
-[connection.ts:93](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L93)
+### dropTable()
 
-___
+> `abstract` **dropTable**(`name`): `Promise`&lt;`void`&gt;
-
-### dropTable
-
-▸ **dropTable**(`name`): `Promise`\<`void`\>
 
 Drop an existing table.
 
 #### Parameters
 
-| Name | Type | Description |
+• **name**: `string`
-| :------ | :------ | :------ |
+
-| `name` | `string` | The name of the table to drop. |
+The name of the table to drop.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[connection.ts:173](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L173)
+### isOpen()
 
-___
+> `abstract` **isOpen**(): `boolean`
-
-### isOpen
-
-▸ **isOpen**(): `boolean`
 
 Return true if the connection has not been closed
 
@@ -188,37 +158,31 @@ Return true if the connection has not been closed
 
 `boolean`
 
-#### Defined in
+***
 
-[connection.ts:77](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L77)
+### openTable()
 
-___
+> `abstract` **openTable**(`name`, `options`?): `Promise`&lt;[`Table`](Table.md)&gt;
-
-### openTable
-
-▸ **openTable**(`name`): `Promise`\<[`Table`](Table.md)\>
 
 Open a table in the database.
 
 #### Parameters
 
-| Name | Type | Description |
+• **name**: `string`
-| :------ | :------ | :------ |
+
-| `name` | `string` | The name of the table |
+The name of the table
+
+• **options?**: `Partial`&lt;`OpenTableOptions`&gt;
 
 #### Returns
 
-`Promise`\<[`Table`](Table.md)\>
+`Promise`&lt;[`Table`](Table.md)&gt;
 
-#### Defined in
+***
 
-[connection.ts:112](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L112)
+### tableNames()
 
-___
+> `abstract` **tableNames**(`options`?): `Promise`&lt;`string`[]&gt;
-
-### tableNames
-
-▸ **tableNames**(`options?`): `Promise`\<`string`[]\>
 
 List all the table names in this database.
 
@@ -226,14 +190,11 @@ Tables will be returned in lexicographical order.
 
 #### Parameters
 
-| Name | Type | Description |
+• **options?**: `Partial`&lt;[`TableNamesOptions`](../interfaces/TableNamesOptions.md)&gt;
-| :------ | :------ | :------ |
+
-| `options?` | `Partial`\<[`TableNamesOptions`](../interfaces/TableNamesOptions.md)\> | options to control the paging / start point |
+options to control the
+paging / start point
 
 #### Returns
 
-`Promise`\<`string`[]\>
+`Promise`&lt;`string`[]&gt;
-
-#### Defined in
-
-[connection.ts:104](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L104)

docs/src/js/classes/Index.md

@@ -1,57 +1,16 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Index
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Index
 
 # Class: Index
 
-## Table of contents
-
-### Constructors
-
-- [constructor](Index.md#constructor)
-
-### Properties
-
-- [inner](Index.md#inner)
-
-### Methods
-
-- [btree](Index.md#btree)
-- [ivfPq](Index.md#ivfpq)
-
-## Constructors
-
-### constructor
-
-• **new Index**(`inner`): [`Index`](Index.md)
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `Index` |
-
-#### Returns
-
-[`Index`](Index.md)
-
-#### Defined in
-
-[indices.ts:118](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L118)
-
-## Properties
-
-### inner
-
-• `Private` `Readonly` **inner**: `Index`
-
-#### Defined in
-
-[indices.ts:117](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L117)
-
 ## Methods
 
-### btree
+### btree()
 
-▸ **btree**(): [`Index`](Index.md)
+> `static` **btree**(): [`Index`](Index.md)
 
 Create a btree index
 
@@ -75,15 +34,11 @@ block size may be added in the future.
 
 [`Index`](Index.md)
 
-#### Defined in
+***
 
-[indices.ts:175](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L175)
+### ivfPq()
 
-___
+> `static` **ivfPq**(`options`?): [`Index`](Index.md)
-
-### ivfPq
-
-▸ **ivfPq**(`options?`): [`Index`](Index.md)
 
 Create an IvfPq index
 
@@ -108,14 +63,8 @@ currently is also a memory intensive operation.
 
 #### Parameters
 
-| Name | Type |
+• **options?**: `Partial`<[`IvfPqOptions`](../interfaces/IvfPqOptions.md)>
-| :------ | :------ |
-| `options?` | `Partial`\<[`IvfPqOptions`](../interfaces/IvfPqOptions.md)\> |
 
 #### Returns
 
 [`Index`](Index.md)
-
-#### Defined in
-
-[indices.ts:144](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L144)

docs/src/js/classes/MakeArrowTableOptions.md

@@ -1,46 +1,32 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / MakeArrowTableOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / MakeArrowTableOptions
 
 # Class: MakeArrowTableOptions
 
 Options to control the makeArrowTable call.
 
-## Table of contents
-
-### Constructors
-
-- [constructor](MakeArrowTableOptions.md#constructor)
-
-### Properties
-
-- [dictionaryEncodeStrings](MakeArrowTableOptions.md#dictionaryencodestrings)
-- [schema](MakeArrowTableOptions.md#schema)
-- [vectorColumns](MakeArrowTableOptions.md#vectorcolumns)
-
 ## Constructors
 
-### constructor
+### new MakeArrowTableOptions()
 
-• **new MakeArrowTableOptions**(`values?`): [`MakeArrowTableOptions`](MakeArrowTableOptions.md)
+> **new MakeArrowTableOptions**(`values`?): [`MakeArrowTableOptions`](MakeArrowTableOptions.md)
 
 #### Parameters
 
-| Name | Type |
+• **values?**: `Partial`<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)>
-| :------ | :------ |
-| `values?` | `Partial`\<[`MakeArrowTableOptions`](MakeArrowTableOptions.md)\> |
 
 #### Returns
 
 [`MakeArrowTableOptions`](MakeArrowTableOptions.md)
 
-#### Defined in
-
-[arrow.ts:100](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L100)
-
 ## Properties
 
 ### dictionaryEncodeStrings
 
-• **dictionaryEncodeStrings**: `boolean` = `false`
+> **dictionaryEncodeStrings**: `boolean` = `false`
 
 If true then string columns will be encoded with dictionary encoding
 
@@ -50,26 +36,26 @@ data type for individual columns.
 
 If `schema` is provided then this property is ignored.
 
-#### Defined in
+***
 
-[arrow.ts:98](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L98)
+### embeddingFunction?
 
-___
+> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)
 
-### schema
+***
 
-• `Optional` **schema**: `Schema`\<`any`\>
+### embeddings?
 
-#### Defined in
+> `optional` **embeddings**: [`EmbeddingFunction`](../namespaces/embedding/classes/EmbeddingFunction.md)&lt;`unknown`, `FunctionOptions`&gt;
 
-[arrow.ts:67](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L67)
+***
 
-___
+### schema?
+
+> `optional` **schema**: `SchemaLike`
+
+***
 
 ### vectorColumns
 
-• **vectorColumns**: `Record`\<`string`, [`VectorColumnOptions`](VectorColumnOptions.md)\>
+> **vectorColumns**: `Record`&lt;`string`, [`VectorColumnOptions`](VectorColumnOptions.md)&gt;
-
-#### Defined in
-
-[arrow.ts:85](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L85)

docs/src/js/classes/Query.md

@@ -1,48 +1,26 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Query
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Query
 
 # Class: Query
 
 A builder for LanceDB queries.
 
-## Hierarchy
+## Extends
 
-- [`QueryBase`](QueryBase.md)\<`NativeQuery`, [`Query`](Query.md)\>
+- [`QueryBase`](QueryBase.md)&lt;`NativeQuery`&gt;
-
-  ↳ **`Query`**
-
-## Table of contents
-
-### Constructors
-
-- [constructor](Query.md#constructor)
-
-### Properties
-
-- [inner](Query.md#inner)
-
-### Methods
-
-- [[asyncIterator]](Query.md#[asynciterator])
-- [execute](Query.md#execute)
-- [limit](Query.md#limit)
-- [nativeExecute](Query.md#nativeexecute)
-- [nearestTo](Query.md#nearestto)
-- [select](Query.md#select)
-- [toArray](Query.md#toarray)
-- [toArrow](Query.md#toarrow)
-- [where](Query.md#where)
 
 ## Constructors
 
-### constructor
+### new Query()
 
-• **new Query**(`tbl`): [`Query`](Query.md)
+> **new Query**(`tbl`): [`Query`](Query.md)
 
 #### Parameters
 
-| Name | Type |
+• **tbl**: `Table`
-| :------ | :------ |
-| `tbl` | `Table` |
 
 #### Returns
 
@@ -50,57 +28,67 @@ A builder for LanceDB queries.
 
 #### Overrides
 
-[QueryBase](QueryBase.md).[constructor](QueryBase.md#constructor)
+[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors)
-
-#### Defined in
-
-[query.ts:329](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L329)
 
 ## Properties
 
 ### inner
 
-• `Protected` **inner**: `Query`
+> `protected` **inner**: `Query` \| `Promise`&lt;`Query`&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[inner](QueryBase.md#inner)
+[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
 
 ## Methods
 
-### [asyncIterator]
+### \[asyncIterator\]()
 
-▸ **[asyncIterator]**(): `AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
+> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
 
 #### Returns
 
-`AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
+`AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[[asyncIterator]](QueryBase.md#[asynciterator])
+[`QueryBase`](QueryBase.md).[`[asyncIterator]`](QueryBase.md#%5Basynciterator%5D)
 
-#### Defined in
+***
 
-[query.ts:154](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L154)
+### doCall()
 
-___
+> `protected` **doCall**(`fn`): `void`
 
-### execute
+#### Parameters
 
-▸ **execute**(): [`RecordBatchIterator`](RecordBatchIterator.md)
+• **fn**
+
+#### Returns
+
+`void`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`doCall`](QueryBase.md#docall)
+
+***
+
+### execute()
+
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
 
 Execute the query and return the results as an
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
 [`RecordBatchIterator`](RecordBatchIterator.md)
 
-**`See`**
+#### See
 
  - AsyncIterator
 of
@@ -114,17 +102,76 @@ single query)
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[execute](QueryBase.md#execute)
+[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
 
-#### Defined in
+***
 
-[query.ts:149](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L149)
+### explainPlan()
 
-___
+> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;
 
-### limit
+Generates an explanation of the query execution plan.
 
-▸ **limit**(`limit`): [`Query`](Query.md)
+#### Parameters
+
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+
+***
+
+### ~~filter()~~
+
+> **filter**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Alias
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
+
+***
+
+### limit()
+
+> **limit**(`limit`): `this`
 
 Set the maximum number of results to return.
 
@@ -133,45 +180,39 @@ called then every valid row from the table will be returned.
 
 #### Parameters
 
-| Name | Type |
+• **limit**: `number`
-| :------ | :------ |
-| `limit` | `number` |
 
 #### Returns
 
-[`Query`](Query.md)
+`this`
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[limit](QueryBase.md#limit)
+[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
 
-#### Defined in
+***
 
-[query.ts:129](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L129)
+### nativeExecute()
 
-___
+> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
 
-### nativeExecute
+#### Parameters
 
-▸ **nativeExecute**(): `Promise`\<`RecordBatchIterator`\>
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
-`Promise`\<`RecordBatchIterator`\>
+`Promise`&lt;`RecordBatchIterator`&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[nativeExecute](QueryBase.md#nativeexecute)
+[`QueryBase`](QueryBase.md).[`nativeExecute`](QueryBase.md#nativeexecute)
 
-#### Defined in
+***
 
-[query.ts:134](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L134)
+### nearestTo()
 
-___
+> **nearestTo**(`vector`): [`VectorQuery`](VectorQuery.md)
-
-### nearestTo
-
-▸ **nearestTo**(`vector`): [`VectorQuery`](VectorQuery.md)
 
 Find the nearest vectors to the given query vector.
 
@@ -191,15 +232,13 @@ If there is more than one vector column you must use
 
 #### Parameters
 
-| Name | Type |
+• **vector**: `IntoVector`
-| :------ | :------ |
-| `vector` | `unknown` |
 
 #### Returns
 
 [`VectorQuery`](VectorQuery.md)
 
-**`See`**
+#### See
 
  - [VectorQuery#column](VectorQuery.md#column)  to specify which column you would like
 to compare with.
@@ -223,15 +262,11 @@ Vector searches always have a `limit`.  If `limit` has not been called then
 a default `limit` of 10 will be used.
  - [Query#limit](Query.md#limit)
 
-#### Defined in
+***
 
-[query.ts:370](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L370)
+### select()
 
-___
+> **select**(`columns`): `this`
-
-### select
-
-▸ **select**(`columns`): [`Query`](Query.md)
 
 Return only the specified columns.
 
@@ -255,15 +290,13 @@ input to this method would be:
 
 #### Parameters
 
-| Name | Type |
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
-| :------ | :------ |
-| `columns` | `string`[] \| `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> |
 
 #### Returns
 
-[`Query`](Query.md)
+`this`
 
-**`Example`**
+#### Example
 
 ```ts
 new Map([["combined", "a + b"], ["c", "c"]])
@@ -278,61 +311,57 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[select](QueryBase.md#select)
+[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
 
-#### Defined in
+***
 
-[query.ts:108](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L108)
+### toArray()
 
-___
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
-
-### toArray
-
-▸ **toArray**(): `Promise`\<`unknown`[]\>
 
 Collect the results as an array of objects.
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
-`Promise`\<`unknown`[]\>
+`Promise`&lt;`any`[]&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[toArray](QueryBase.md#toarray)
+[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
 
-#### Defined in
+***
 
-[query.ts:169](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L169)
+### toArrow()
 
-___
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
-
-### toArrow
-
-▸ **toArrow**(): `Promise`\<`Table`\<`any`\>\>
 
 Collect the results as an Arrow
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
-`Promise`\<`Table`\<`any`\>\>
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
 
-**`See`**
+#### See
 
 ArrowTable.
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[toArrow](QueryBase.md#toarrow)
+[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
 
-#### Defined in
+***
 
-[query.ts:160](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L160)
+### where()
 
-___
+> **where**(`predicate`): `this`
-
-### where
-
-▸ **where**(`predicate`): [`Query`](Query.md)
 
 A filter statement to be applied to this query.
 
@@ -340,15 +369,13 @@ The filter should be supplied as an SQL query string.  For example:
 
 #### Parameters
 
-| Name | Type |
+• **predicate**: `string`
-| :------ | :------ |
-| `predicate` | `string` |
 
 #### Returns
 
-[`Query`](Query.md)
+`this`
 
-**`Example`**
+#### Example
 
 ```ts
 x > 10
@@ -361,8 +388,4 @@ on the filter column(s).
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[where](QueryBase.md#where)
+[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
-
-#### Defined in
-
-[query.ts:73](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L73)

docs/src/js/classes/QueryBase.md

@@ -1,117 +1,91 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / QueryBase
+[**@lancedb/lancedb**](../README.md) • **Docs**
 
-# Class: QueryBase\<NativeQueryType, QueryType\>
+***
+
+[@lancedb/lancedb](../globals.md) / QueryBase
+
+# Class: QueryBase&lt;NativeQueryType&gt;
 
 Common methods supported by all query types
 
-## Type parameters
+## Extended by
 
-| Name | Type |
+- [`Query`](Query.md)
-| :------ | :------ |
+- [`VectorQuery`](VectorQuery.md)
-| `NativeQueryType` | extends `NativeQuery` \| `NativeVectorQuery` |
-| `QueryType` | `QueryType` |
 
-## Hierarchy
+## Type Parameters
 
-- **`QueryBase`**
+• **NativeQueryType** *extends* `NativeQuery` \| `NativeVectorQuery`
-
-  ↳ [`Query`](Query.md)
-
-  ↳ [`VectorQuery`](VectorQuery.md)
 
 ## Implements
 
-- `AsyncIterable`\<`RecordBatch`\>
+- `AsyncIterable`&lt;`RecordBatch`&gt;
-
-## Table of contents
-
-### Constructors
-
-- [constructor](QueryBase.md#constructor)
-
-### Properties
-
-- [inner](QueryBase.md#inner)
-
-### Methods
-
-- [[asyncIterator]](QueryBase.md#[asynciterator])
-- [execute](QueryBase.md#execute)
-- [limit](QueryBase.md#limit)
-- [nativeExecute](QueryBase.md#nativeexecute)
-- [select](QueryBase.md#select)
-- [toArray](QueryBase.md#toarray)
-- [toArrow](QueryBase.md#toarrow)
-- [where](QueryBase.md#where)
 
 ## Constructors
 
-### constructor
+### new QueryBase()
 
-• **new QueryBase**\<`NativeQueryType`, `QueryType`\>(`inner`): [`QueryBase`](QueryBase.md)\<`NativeQueryType`, `QueryType`\>
+> `protected` **new QueryBase**&lt;`NativeQueryType`&gt;(`inner`): [`QueryBase`](QueryBase.md)&lt;`NativeQueryType`&gt;
-
-#### Type parameters
-
-| Name | Type |
-| :------ | :------ |
-| `NativeQueryType` | extends `Query` \| `VectorQuery` |
-| `QueryType` | `QueryType` |
 
 #### Parameters
 
-| Name | Type |
+• **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;
-| :------ | :------ |
-| `inner` | `NativeQueryType` |
 
 #### Returns
 
-[`QueryBase`](QueryBase.md)\<`NativeQueryType`, `QueryType`\>
+[`QueryBase`](QueryBase.md)&lt;`NativeQueryType`&gt;
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
 
 ## Properties
 
 ### inner
 
-• `Protected` **inner**: `NativeQueryType`
+> `protected` **inner**: `NativeQueryType` \| `Promise`&lt;`NativeQueryType`&gt;
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
 
 ## Methods
 
-### [asyncIterator]
+### \[asyncIterator\]()
 
-▸ **[asyncIterator]**(): `AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
+> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
 
 #### Returns
 
-`AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
+`AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
 
 #### Implementation of
 
-AsyncIterable.[asyncIterator]
+`AsyncIterable.[asyncIterator]`
 
-#### Defined in
+***
 
-[query.ts:154](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L154)
+### doCall()
 
-___
+> `protected` **doCall**(`fn`): `void`
 
-### execute
+#### Parameters
 
-▸ **execute**(): [`RecordBatchIterator`](RecordBatchIterator.md)
+• **fn**
+
+#### Returns
+
+`void`
+
+***
+
+### execute()
+
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
 
 Execute the query and return the results as an
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
 [`RecordBatchIterator`](RecordBatchIterator.md)
 
-**`See`**
+#### See
 
  - AsyncIterator
 of
@@ -123,15 +97,66 @@ This readahead is limited however and backpressure will be applied if this
 stream is consumed slowly (this constrains the maximum memory used by a
 single query)
 
-#### Defined in
+***
 
-[query.ts:149](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L149)
+### explainPlan()
 
-___
+> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;
 
-### limit
+Generates an explanation of the query execution plan.
 
-▸ **limit**(`limit`): `QueryType`
+#### Parameters
+
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+***
+
+### ~~filter()~~
+
+> **filter**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Alias
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+***
+
+### limit()
+
+> **limit**(`limit`): `this`
 
 Set the maximum number of results to return.
 
@@ -140,37 +165,31 @@ called then every valid row from the table will be returned.
 
 #### Parameters
 
-| Name | Type |
+• **limit**: `number`
-| :------ | :------ |
-| `limit` | `number` |
 
 #### Returns
 
-`QueryType`
+`this`
 
-#### Defined in
+***
 
-[query.ts:129](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L129)
+### nativeExecute()
 
-___
+> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
 
-### nativeExecute
+#### Parameters
 
-▸ **nativeExecute**(): `Promise`\<`RecordBatchIterator`\>
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
-`Promise`\<`RecordBatchIterator`\>
+`Promise`&lt;`RecordBatchIterator`&gt;
 
-#### Defined in
+***
 
-[query.ts:134](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L134)
+### select()
 
-___
+> **select**(`columns`): `this`
-
-### select
-
-▸ **select**(`columns`): `QueryType`
 
 Return only the specified columns.
 
@@ -194,15 +213,13 @@ input to this method would be:
 
 #### Parameters
 
-| Name | Type |
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
-| :------ | :------ |
-| `columns` | `string`[] \| `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> |
 
 #### Returns
 
-`QueryType`
+`this`
 
-**`Example`**
+#### Example
 
 ```ts
 new Map([["combined", "a + b"], ["c", "c"]])
@@ -215,51 +232,47 @@ uses `Object.entries` which should preserve the insertion order of the object.
 object insertion order is easy to get wrong and `Map` is more foolproof.
 ```
 
-#### Defined in
+***
 
-[query.ts:108](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L108)
+### toArray()
 
-___
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
-
-### toArray
-
-▸ **toArray**(): `Promise`\<`unknown`[]\>
 
 Collect the results as an array of objects.
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
-`Promise`\<`unknown`[]\>
+`Promise`&lt;`any`[]&gt;
 
-#### Defined in
+***
 
-[query.ts:169](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L169)
+### toArrow()
 
-___
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
-
-### toArrow
-
-▸ **toArrow**(): `Promise`\<`Table`\<`any`\>\>
 
 Collect the results as an Arrow
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
-`Promise`\<`Table`\<`any`\>\>
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
 
-**`See`**
+#### See
 
 ArrowTable.
 
-#### Defined in
+***
 
-[query.ts:160](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L160)
+### where()
 
-___
+> **where**(`predicate`): `this`
-
-### where
-
-▸ **where**(`predicate`): `QueryType`
 
 A filter statement to be applied to this query.
 
@@ -267,15 +280,13 @@ The filter should be supplied as an SQL query string.  For example:
 
 #### Parameters
 
-| Name | Type |
+• **predicate**: `string`
-| :------ | :------ |
-| `predicate` | `string` |
 
 #### Returns
 
-`QueryType`
+`this`
 
-**`Example`**
+#### Example
 
 ```ts
 x > 10
@@ -285,7 +296,3 @@ x > 5 OR y = 'test'
 Filtering performance can often be improved by creating a scalar index
 on the filter column(s).
 ```
-
-#### Defined in
-
-[query.ts:73](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L73)

docs/src/js/classes/RecordBatchIterator.md

@@ -1,80 +1,39 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / RecordBatchIterator
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / RecordBatchIterator
 
 # Class: RecordBatchIterator
 
 ## Implements
 
-- `AsyncIterator`\<`RecordBatch`\>
+- `AsyncIterator`&lt;`RecordBatch`&gt;
-
-## Table of contents
-
-### Constructors
-
-- [constructor](RecordBatchIterator.md#constructor)
-
-### Properties
-
-- [inner](RecordBatchIterator.md#inner)
-- [promisedInner](RecordBatchIterator.md#promisedinner)
-
-### Methods
-
-- [next](RecordBatchIterator.md#next)
 
 ## Constructors
 
-### constructor
+### new RecordBatchIterator()
 
-• **new RecordBatchIterator**(`promise?`): [`RecordBatchIterator`](RecordBatchIterator.md)
+> **new RecordBatchIterator**(`promise`?): [`RecordBatchIterator`](RecordBatchIterator.md)
 
 #### Parameters
 
-| Name | Type |
+• **promise?**: `Promise`&lt;`RecordBatchIterator`&gt;
-| :------ | :------ |
-| `promise?` | `Promise`\<`RecordBatchIterator`\> |
 
 #### Returns
 
 [`RecordBatchIterator`](RecordBatchIterator.md)
 
-#### Defined in
-
-[query.ts:27](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L27)
-
-## Properties
-
-### inner
-
-• `Private` `Optional` **inner**: `RecordBatchIterator`
-
-#### Defined in
-
-[query.ts:25](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L25)
-
-___
-
-### promisedInner
-
-• `Private` `Optional` **promisedInner**: `Promise`\<`RecordBatchIterator`\>
-
-#### Defined in
-
-[query.ts:24](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L24)
-
 ## Methods
 
-### next
+### next()
 
-▸ **next**(): `Promise`\<`IteratorResult`\<`RecordBatch`\<`any`\>, `any`\>\>
+> **next**(): `Promise`&lt;`IteratorResult`&lt;`RecordBatch`&lt;`any`&gt;, `any`&gt;&gt;
 
 #### Returns
 
-`Promise`\<`IteratorResult`\<`RecordBatch`\<`any`\>, `any`\>\>
+`Promise`&lt;`IteratorResult`&lt;`RecordBatch`&lt;`any`&gt;, `any`&gt;&gt;
 
 #### Implementation of
 
-AsyncIterator.next
+`AsyncIterator.next`
-
-#### Defined in
-
-[query.ts:33](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L33)

docs/src/js/classes/Table.md

@@ -1,6 +1,10 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / Table
+[**@lancedb/lancedb**](../README.md) • **Docs**
 
-# Class: Table
+***
+
+[@lancedb/lancedb](../globals.md) / Table
+
+# Class: `abstract` Table
 
 A Table is a collection of Records in a LanceDB Database.
 
@@ -13,196 +17,149 @@ further operations.
 Closing a table is optional.  It not closed, it will be closed when it is garbage
 collected.
 
-## Table of contents
-
-### Constructors
-
-- [constructor](Table.md#constructor)
-
-### Properties
-
-- [inner](Table.md#inner)
-
-### Methods
-
-- [add](Table.md#add)
-- [addColumns](Table.md#addcolumns)
-- [alterColumns](Table.md#altercolumns)
-- [checkout](Table.md#checkout)
-- [checkoutLatest](Table.md#checkoutlatest)
-- [close](Table.md#close)
-- [countRows](Table.md#countrows)
-- [createIndex](Table.md#createindex)
-- [delete](Table.md#delete)
-- [display](Table.md#display)
-- [dropColumns](Table.md#dropcolumns)
-- [isOpen](Table.md#isopen)
-- [listIndices](Table.md#listindices)
-- [query](Table.md#query)
-- [restore](Table.md#restore)
-- [schema](Table.md#schema)
-- [update](Table.md#update)
-- [vectorSearch](Table.md#vectorsearch)
-- [version](Table.md#version)
-
 ## Constructors
 
-### constructor
+### new Table()
 
-• **new Table**(`inner`): [`Table`](Table.md)
+> **new Table**(): [`Table`](Table.md)
-
-Construct a Table. Internal use only.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `inner` | `Table` |
 
 #### Returns
 
 [`Table`](Table.md)
 
-#### Defined in
+## Accessors
 
-[table.ts:69](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L69)
+### name
 
-## Properties
+> `get` `abstract` **name**(): `string`
 
-### inner
+Returns the name of the table
 
-• `Private` `Readonly` **inner**: `Table`
+#### Returns
 
-#### Defined in
+`string`
-
-[table.ts:66](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L66)
 
 ## Methods
 
-### add
+### add()
 
-▸ **add**(`data`, `options?`): `Promise`\<`void`\>
+> `abstract` **add**(`data`, `options`?): `Promise`&lt;`void`&gt;
 
 Insert records into this Table.
 
 #### Parameters
 
-| Name | Type | Description |
+• **data**: [`Data`](../type-aliases/Data.md)
-| :------ | :------ | :------ |
+
-| `data` | [`Data`](../modules.md#data) | Records to be inserted into the Table |
+Records to be inserted into the Table
-| `options?` | `Partial`\<[`AddDataOptions`](../interfaces/AddDataOptions.md)\> | - |
+
+• **options?**: `Partial`&lt;[`AddDataOptions`](../interfaces/AddDataOptions.md)&gt;
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:105](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L105)
+### addColumns()
 
-___
+> `abstract` **addColumns**(`newColumnTransforms`): `Promise`&lt;`void`&gt;
-
-### addColumns
-
-▸ **addColumns**(`newColumnTransforms`): `Promise`\<`void`\>
 
 Add new columns with defined values.
 
 #### Parameters
 
-| Name | Type | Description |
+• **newColumnTransforms**: [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[]
-| :------ | :------ | :------ |
+
-| `newColumnTransforms` | [`AddColumnsSql`](../interfaces/AddColumnsSql.md)[] | pairs of column names and the SQL expression to use to calculate the value of the new column. These expressions will be evaluated for each row in the table, and can reference existing columns in the table. |
+pairs of column names and
+the SQL expression to use to calculate the value of the new column. These
+expressions will be evaluated for each row in the table, and can
+reference existing columns in the table.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:261](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L261)
+### alterColumns()
 
-___
+> `abstract` **alterColumns**(`columnAlterations`): `Promise`&lt;`void`&gt;
-
-### alterColumns
-
-▸ **alterColumns**(`columnAlterations`): `Promise`\<`void`\>
 
 Alter the name or nullability of columns.
 
 #### Parameters
 
-| Name | Type | Description |
+• **columnAlterations**: [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[]
-| :------ | :------ | :------ |
+
-| `columnAlterations` | [`ColumnAlteration`](../interfaces/ColumnAlteration.md)[] | One or more alterations to apply to columns. |
+One or more alterations to
+apply to columns.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:270](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L270)
+### checkout()
 
-___
+> `abstract` **checkout**(`version`): `Promise`&lt;`void`&gt;
 
-### checkout
+Checks out a specific version of the table _This is an in-place operation._
 
-▸ **checkout**(`version`): `Promise`\<`void`\>
+This allows viewing previous versions of the table. If you wish to
+keep writing to the dataset starting from an old version, then use
+the `restore` function.
 
-Checks out a specific version of the Table
+Calling this method will set the table into time-travel mode. If you
-
+wish to return to standard mode, call `checkoutLatest`.
-Any read operation on the table will now access the data at the checked out version.
-As a consequence, calling this method will disable any read consistency interval
-that was previously set.
-
-This is a read-only operation that turns the table into a sort of "view"
-or "detached head".  Other table instances will not be affected.  To make the change
-permanent you can use the `[Self::restore]` method.
-
-Any operation that modifies the table will fail while the table is in a checked
-out state.
-
-To return the table to a normal state use `[Self::checkout_latest]`
 
 #### Parameters
 
-| Name | Type |
+• **version**: `number`
-| :------ | :------ |
+
-| `version` | `number` |
+The version to checkout
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+#### Example
 
-[table.ts:317](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L317)
+```typescript
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], type: "vector" },
+]);
 
-___
+console.log(await table.version()); // 1
+console.log(table.display());
+await table.add([{ vector: [0.5, 0.2], type: "vector" }]);
+await table.checkout(1);
+console.log(await table.version()); // 2
+```
 
-### checkoutLatest
+***
 
-▸ **checkoutLatest**(): `Promise`\<`void`\>
+### checkoutLatest()
 
-Ensures the table is pointing at the latest version
+> `abstract` **checkoutLatest**(): `Promise`&lt;`void`&gt;
 
-This can be used to manually update a table when the read_consistency_interval is None
+Checkout the latest version of the table. _This is an in-place operation._
-It can also be used to undo a `[Self::checkout]` operation
+
+The table will be set back into standard mode, and will track the latest
+version of the table.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:327](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L327)
+### close()
 
-___
+> `abstract` **close**(): `void`
-
-### close
-
-▸ **close**(): `void`
 
 Close the table, releasing any underlying resources.
 
@@ -214,37 +171,27 @@ Any attempt to use the table after it is closed will result in an error.
 
 `void`
 
-#### Defined in
+***
 
-[table.ts:85](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L85)
+### countRows()
 
-___
+> `abstract` **countRows**(`filter`?): `Promise`&lt;`number`&gt;
-
-### countRows
-
-▸ **countRows**(`filter?`): `Promise`\<`number`\>
 
 Count the total number of rows in the dataset.
 
 #### Parameters
 
-| Name | Type |
+• **filter?**: `string`
-| :------ | :------ |
-| `filter?` | `string` |
 
 #### Returns
 
-`Promise`\<`number`\>
+`Promise`&lt;`number`&gt;
 
-#### Defined in
+***
 
-[table.ts:152](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L152)
+### createIndex()
 
-___
+> `abstract` **createIndex**(`column`, `options`?): `Promise`&lt;`void`&gt;
-
-### createIndex
-
-▸ **createIndex**(`column`, `options?`): `Promise`\<`void`\>
 
 Create an index to speed up queries.
 
@@ -255,73 +202,66 @@ vector and non-vector searches)
 
 #### Parameters
 
-| Name | Type |
+• **column**: `string`
-| :------ | :------ |
+
-| `column` | `string` |
+• **options?**: `Partial`&lt;[`IndexOptions`](../interfaces/IndexOptions.md)&gt;
-| `options?` | `Partial`\<[`IndexOptions`](../interfaces/IndexOptions.md)\> |
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-**`Example`**
+#### Note
+
+We currently don't support custom named indexes,
+The index name will always be `${column}_idx`
+
+#### Examples
 
 ```ts
 // If the column has a vector (fixed size list) data type then
 // an IvfPq vector index will be created.
 const table = await conn.openTable("my_table");
-await table.createIndex(["vector"]);
+await table.createIndex("vector");
 ```
 
-**`Example`**
-
 ```ts
 // For advanced control over vector index creation you can specify
 // the index type and options.
 const table = await conn.openTable("my_table");
-await table.createIndex(["vector"], I)
+await table.createIndex("vector", {
-  .ivf_pq({ num_partitions: 128, num_sub_vectors: 16 })
+  config: lancedb.Index.ivfPq({
-  .build();
+    numPartitions: 128,
+    numSubVectors: 16,
+  }),
+});
 ```
 
-**`Example`**
-
 ```ts
 // Or create a Scalar index
-await table.createIndex("my_float_col").build();
+await table.createIndex("my_float_col");
 ```
 
-#### Defined in
+***
 
-[table.ts:184](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L184)
+### delete()
 
-___
+> `abstract` **delete**(`predicate`): `Promise`&lt;`void`&gt;
-
-### delete
-
-▸ **delete**(`predicate`): `Promise`\<`void`\>
 
 Delete the rows that satisfy the predicate.
 
 #### Parameters
 
-| Name | Type |
+• **predicate**: `string`
-| :------ | :------ |
-| `predicate` | `string` |
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:157](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L157)
+### display()
 
-___
+> `abstract` **display**(): `string`
-
-### display
-
-▸ **display**(): `string`
 
 Return a brief description of the table
 
@@ -329,15 +269,11 @@ Return a brief description of the table
 
 `string`
 
-#### Defined in
+***
 
-[table.ts:90](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L90)
+### dropColumns()
 
-___
+> `abstract` **dropColumns**(`columnNames`): `Promise`&lt;`void`&gt;
-
-### dropColumns
-
-▸ **dropColumns**(`columnNames`): `Promise`\<`void`\>
 
 Drop one or more columns from the dataset
 
@@ -348,23 +284,41 @@ then call ``cleanup_files`` to remove the old files.
 
 #### Parameters
 
-| Name | Type | Description |
+• **columnNames**: `string`[]
-| :------ | :------ | :------ |
+
-| `columnNames` | `string`[] | The names of the columns to drop. These can be nested column references (e.g. "a.b.c") or top-level column names (e.g. "a"). |
+The names of the columns to drop. These can
+be nested column references (e.g. "a.b.c") or top-level column names
+(e.g. "a").
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:285](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L285)
+### indexStats()
 
-___
+> `abstract` **indexStats**(`name`): `Promise`&lt;`undefined` \| [`IndexStatistics`](../interfaces/IndexStatistics.md)&gt;
 
-### isOpen
+List all the stats of a specified index
 
-▸ **isOpen**(): `boolean`
+#### Parameters
+
+• **name**: `string`
+
+The name of the index.
+
+#### Returns
+
+`Promise`&lt;`undefined` \| [`IndexStatistics`](../interfaces/IndexStatistics.md)&gt;
+
+The stats of the index. If the index does not exist, it will return undefined
+
+***
+
+### isOpen()
+
+> `abstract` **isOpen**(): `boolean`
 
 Return true if the table has not been closed
 
@@ -372,31 +326,79 @@ Return true if the table has not been closed
 
 `boolean`
 
-#### Defined in
+***
 
-[table.ts:74](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L74)
+### listIndices()
 
-___
+> `abstract` **listIndices**(): `Promise`&lt;[`IndexConfig`](../interfaces/IndexConfig.md)[]&gt;
 
-### listIndices
+List all indices that have been created with [Table.createIndex](Table.md#createindex)
-
-▸ **listIndices**(): `Promise`\<[`IndexConfig`](../interfaces/IndexConfig.md)[]\>
-
-List all indices that have been created with Self::create_index
 
 #### Returns
 
-`Promise`\<[`IndexConfig`](../interfaces/IndexConfig.md)[]\>
+`Promise`&lt;[`IndexConfig`](../interfaces/IndexConfig.md)[]&gt;
 
-#### Defined in
+***
 
-[table.ts:350](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L350)
+### mergeInsert()
 
-___
+> `abstract` **mergeInsert**(`on`): `MergeInsertBuilder`
 
-### query
+#### Parameters
 
-▸ **query**(): [`Query`](Query.md)
+• **on**: `string` \| `string`[]
+
+#### Returns
+
+`MergeInsertBuilder`
+
+***
+
+### optimize()
+
+> `abstract` **optimize**(`options`?): `Promise`&lt;`OptimizeStats`&gt;
+
+Optimize the on-disk data and indices for better performance.
+
+Modeled after ``VACUUM`` in PostgreSQL.
+
+ Optimization covers three operations:
+
+ - Compaction: Merges small files into larger ones
+ - Prune: Removes old versions of the dataset
+ - Index: Optimizes the indices, adding new data to existing indices
+
+ Experimental API
+ ----------------
+
+ The optimization process is undergoing active development and may change.
+ Our goal with these changes is to improve the performance of optimization and
+ reduce the complexity.
+
+ That being said, it is essential today to run optimize if you want the best
+ performance.  It should be stable and safe to use in production, but it our
+ hope that the API may be simplified (or not even need to be called) in the
+ future.
+
+ The frequency an application shoudl call optimize is based on the frequency of
+ data modifications.  If data is frequently added, deleted, or updated then
+ optimize should be run frequently.  A good rule of thumb is to run optimize if
+ you have added or modified 100,000 or more records or run more than 20 data
+ modification operations.
+
+#### Parameters
+
+• **options?**: `Partial`&lt;`OptimizeOptions`&gt;
+
+#### Returns
+
+`Promise`&lt;`OptimizeStats`&gt;
+
+***
+
+### query()
+
+> `abstract` **query**(): [`Query`](Query.md)
 
 Create a [Query](Query.md) Builder.
 
@@ -406,8 +408,7 @@ returned by this method can be used to control the query using filtering,
 vector similarity, sorting, and more.
 
 Note: By default, all columns are returned.  For best performance, you should
-only fetch the columns you need.  See [`Query::select_with_projection`] for
+only fetch the columns you need.
-more details.
 
 When appropriate, various indices and statistics based pruning will be used to
 accelerate the query.
@@ -418,21 +419,22 @@ accelerate the query.
 
 A builder that can be used to parameterize the query
 
-**`Example`**
+#### Examples
 
 ```ts
 // SQL-style filtering
 //
 // This query will return up to 1000 rows whose value in the `id` column
-// is greater than 5.  LanceDb supports a broad set of filtering functions.
+// is greater than 5. LanceDb supports a broad set of filtering functions.
-for await (const batch of table.query()
+for await (const batch of table
-                         .filter("id > 1").select(["id"]).limit(20)) {
+  .query()
- console.log(batch);
+  .where("id > 1")
+  .select(["id"])
+  .limit(20)) {
+  console.log(batch);
 }
 ```
 
-**`Example`**
-
 ```ts
 // Vector Similarity Search
 //
@@ -440,18 +442,17 @@ for await (const batch of table.query()
 // closest to the query vector [1.0, 2.0, 3.0].  If an index has been created
 // on the "vector" column then this will perform an ANN search.
 //
-// The `refine_factor` and `nprobes` methods are used to control the recall /
+// The `refineFactor` and `nprobes` methods are used to control the recall /
 // latency tradeoff of the search.
-for await (const batch of table.query()
+for await (const batch of table
-                   .nearestTo([1, 2, 3])
+  .query()
-                   .refineFactor(5).nprobe(10)
+  .where("id > 1")
-                   .limit(10)) {
+  .select(["id"])
- console.log(batch);
+  .limit(20)) {
+  console.log(batch);
 }
 ```
 
-**`Example`**
-
 ```ts
 // Scan the full dataset
 //
@@ -461,15 +462,11 @@ for await (const batch of table.query()) {
 }
 ```
 
-#### Defined in
+***
 
-[table.ts:238](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L238)
+### restore()
 
-___
+> `abstract` **restore**(): `Promise`&lt;`void`&gt;
-
-### restore
-
-▸ **restore**(): `Promise`\<`void`\>
 
 Restore the table to the currently checked out version
 
@@ -484,33 +481,121 @@ out state and the read_consistency_interval, if any, will apply.
 
 #### Returns
 
-`Promise`\<`void`\>
+`Promise`&lt;`void`&gt;
 
-#### Defined in
+***
 
-[table.ts:343](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L343)
+### schema()
 
-___
+> `abstract` **schema**(): `Promise`&lt;`Schema`&lt;`any`&gt;&gt;
-
-### schema
-
-▸ **schema**(): `Promise`\<`Schema`\<`any`\>\>
 
 Get the schema of the table.
 
 #### Returns
 
-`Promise`\<`Schema`\<`any`\>\>
+`Promise`&lt;`Schema`&lt;`any`&gt;&gt;
 
-#### Defined in
+***
 
-[table.ts:95](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L95)
+### search()
 
-___
+#### search(query)
 
-### update
+> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
 
-▸ **update**(`updates`, `options?`): `Promise`\<`void`\>
+Create a search query to find the nearest neighbors
+of the given query vector
+
+##### Parameters
+
+• **query**: `string`
+
+the query. This will be converted to a vector using the table's provided embedding function
+
+##### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+##### Note
+
+If no embedding functions are defined in the table, this will error when collecting the results.
+
+#### search(query)
+
+> `abstract` **search**(`query`): [`VectorQuery`](VectorQuery.md)
+
+Create a search query to find the nearest neighbors
+of the given query vector
+
+##### Parameters
+
+• **query**: `IntoVector`
+
+the query vector
+
+##### Returns
+
+[`VectorQuery`](VectorQuery.md)
+
+***
+
+### toArrow()
+
+> `abstract` **toArrow**(): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+Return the table as an arrow table
+
+#### Returns
+
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
+
+***
+
+### update()
+
+#### update(opts)
+
+> `abstract` **update**(`opts`): `Promise`&lt;`void`&gt;
+
+Update existing records in the Table
+
+##### Parameters
+
+• **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+
+##### Returns
+
+`Promise`&lt;`void`&gt;
+
+##### Example
+
+```ts
+table.update({where:"x = 2", values:{"vector": [10, 10]}})
+```
+
+#### update(opts)
+
+> `abstract` **update**(`opts`): `Promise`&lt;`void`&gt;
+
+Update existing records in the Table
+
+##### Parameters
+
+• **opts**: `object` & `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
+
+##### Returns
+
+`Promise`&lt;`void`&gt;
+
+##### Example
+
+```ts
+table.update({where:"x = 2", valuesSql:{"x": "x + 1"}})
+```
+
+#### update(updates, options)
+
+> `abstract` **update**(`updates`, `options`?): `Promise`&lt;`void`&gt;
 
 Update existing records in the Table
 
@@ -527,26 +612,32 @@ you are updating many rows (with different ids) then you will get
 better performance with a single [`merge_insert`] call instead of
 repeatedly calilng this method.
 
-#### Parameters
+##### Parameters
 
-| Name | Type | Description |
+• **updates**: `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
-| :------ | :------ | :------ |
-| `updates` | `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> | the columns to update Keys in the map should specify the name of the column to update. Values in the map provide the new value of the column. These can be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions based on the row being updated (e.g. "my_col + 1") |
-| `options?` | `Partial`\<[`UpdateOptions`](../interfaces/UpdateOptions.md)\> | additional options to control the update behavior |
 
-#### Returns
+the
+columns to update
 
-`Promise`\<`void`\>
+Keys in the map should specify the name of the column to update.
+Values in the map provide the new value of the column.  These can
+be SQL literal strings (e.g. "7" or "'foo'") or they can be expressions
+based on the row being updated (e.g. "my_col + 1")
 
-#### Defined in
+• **options?**: `Partial`&lt;[`UpdateOptions`](../interfaces/UpdateOptions.md)&gt;
 
-[table.ts:137](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L137)
+additional options to control
+the update behavior
 
-___
+##### Returns
 
-### vectorSearch
+`Promise`&lt;`void`&gt;
 
-▸ **vectorSearch**(`vector`): [`VectorQuery`](VectorQuery.md)
+***
+
+### vectorSearch()
+
+> `abstract` **vectorSearch**(`vector`): [`VectorQuery`](VectorQuery.md)
 
 Search the table with a given query vector.
 
@@ -556,39 +647,50 @@ by `query`.
 
 #### Parameters
 
-| Name | Type |
+• **vector**: `IntoVector`
-| :------ | :------ |
-| `vector` | `unknown` |
 
 #### Returns
 
 [`VectorQuery`](VectorQuery.md)
 
-**`See`**
+#### See
 
 [Query#nearestTo](Query.md#nearestto) for more details.
 
-#### Defined in
+***
 
-[table.ts:249](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L249)
+### version()
 
-___
+> `abstract` **version**(): `Promise`&lt;`number`&gt;
-
-### version
-
-▸ **version**(): `Promise`\<`number`\>
 
 Retrieve the version of the table
 
-LanceDb supports versioning.  Every operation that modifies the table increases
+#### Returns
-version.  As long as a version hasn't been deleted you can `[Self::checkout]` that
+
-version to view the data at that point.  In addition, you can `[Self::restore]` the
+`Promise`&lt;`number`&gt;
-version to replace the current table with a previous version.
+
+***
+
+### parseTableData()
+
+> `static` **parseTableData**(`data`, `options`?, `streaming`?): `Promise`&lt;`object`&gt;
+
+#### Parameters
+
+• **data**: `TableLike` \| `Record`&lt;`string`, `unknown`&gt;[]
+
+• **options?**: `Partial`&lt;[`CreateTableOptions`](../interfaces/CreateTableOptions.md)&gt;
+
+• **streaming?**: `boolean` = `false`
 
 #### Returns
 
-`Promise`\<`number`\>
+`Promise`&lt;`object`&gt;
 
-#### Defined in
+##### buf
 
-[table.ts:297](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L297)
+> **buf**: `Buffer`
+
+##### mode
+
+> **mode**: `string`

docs/src/js/classes/VectorColumnOptions.md

@@ -1,45 +1,29 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / VectorColumnOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / VectorColumnOptions
 
 # Class: VectorColumnOptions
 
-## Table of contents
-
-### Constructors
-
-- [constructor](VectorColumnOptions.md#constructor)
-
-### Properties
-
-- [type](VectorColumnOptions.md#type)
-
 ## Constructors
 
-### constructor
+### new VectorColumnOptions()
 
-• **new VectorColumnOptions**(`values?`): [`VectorColumnOptions`](VectorColumnOptions.md)
+> **new VectorColumnOptions**(`values`?): [`VectorColumnOptions`](VectorColumnOptions.md)
 
 #### Parameters
 
-| Name | Type |
+• **values?**: `Partial`<[`VectorColumnOptions`](VectorColumnOptions.md)>
-| :------ | :------ |
-| `values?` | `Partial`\<[`VectorColumnOptions`](VectorColumnOptions.md)\> |
 
 #### Returns
 
 [`VectorColumnOptions`](VectorColumnOptions.md)
 
-#### Defined in
-
-[arrow.ts:49](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L49)
-
 ## Properties
 
 ### type
 
-• **type**: `Float`\<`Floats`\>
+> **type**: `Float`&lt;`Floats`&gt;
 
 Vector column type.
-
-#### Defined in
-
-[arrow.ts:47](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L47)

docs/src/js/classes/VectorQuery.md

@@ -1,4 +1,8 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / VectorQuery
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / VectorQuery
 
 # Class: VectorQuery
 
@@ -6,50 +10,19 @@ A builder used to construct a vector search
 
 This builder can be reused to execute the query many times.
 
-## Hierarchy
+## Extends
 
-- [`QueryBase`](QueryBase.md)\<`NativeVectorQuery`, [`VectorQuery`](VectorQuery.md)\>
+- [`QueryBase`](QueryBase.md)&lt;`NativeVectorQuery`&gt;
-
-  ↳ **`VectorQuery`**
-
-## Table of contents
-
-### Constructors
-
-- [constructor](VectorQuery.md#constructor)
-
-### Properties
-
-- [inner](VectorQuery.md#inner)
-
-### Methods
-
-- [[asyncIterator]](VectorQuery.md#[asynciterator])
-- [bypassVectorIndex](VectorQuery.md#bypassvectorindex)
-- [column](VectorQuery.md#column)
-- [distanceType](VectorQuery.md#distancetype)
-- [execute](VectorQuery.md#execute)
-- [limit](VectorQuery.md#limit)
-- [nativeExecute](VectorQuery.md#nativeexecute)
-- [nprobes](VectorQuery.md#nprobes)
-- [postfilter](VectorQuery.md#postfilter)
-- [refineFactor](VectorQuery.md#refinefactor)
-- [select](VectorQuery.md#select)
-- [toArray](VectorQuery.md#toarray)
-- [toArrow](VectorQuery.md#toarrow)
-- [where](VectorQuery.md#where)
 
 ## Constructors
 
-### constructor
+### new VectorQuery()
 
-• **new VectorQuery**(`inner`): [`VectorQuery`](VectorQuery.md)
+> **new VectorQuery**(`inner`): [`VectorQuery`](VectorQuery.md)
 
 #### Parameters
 
-| Name | Type |
+• **inner**: `VectorQuery` \| `Promise`&lt;`VectorQuery`&gt;
-| :------ | :------ |
-| `inner` | `VectorQuery` |
 
 #### Returns
 
@@ -57,49 +30,37 @@ This builder can be reused to execute the query many times.
 
 #### Overrides
 
-[QueryBase](QueryBase.md).[constructor](QueryBase.md#constructor)
+[`QueryBase`](QueryBase.md).[`constructor`](QueryBase.md#constructors)
-
-#### Defined in
-
-[query.ts:189](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L189)
 
 ## Properties
 
 ### inner
 
-• `Protected` **inner**: `VectorQuery`
+> `protected` **inner**: `VectorQuery` \| `Promise`&lt;`VectorQuery`&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[inner](QueryBase.md#inner)
+[`QueryBase`](QueryBase.md).[`inner`](QueryBase.md#inner)
-
-#### Defined in
-
-[query.ts:59](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L59)
 
 ## Methods
 
-### [asyncIterator]
+### \[asyncIterator\]()
 
-▸ **[asyncIterator]**(): `AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
+> **\[asyncIterator\]**(): `AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
 
 #### Returns
 
-`AsyncIterator`\<`RecordBatch`\<`any`\>, `any`, `undefined`\>
+`AsyncIterator`&lt;`RecordBatch`&lt;`any`&gt;, `any`, `undefined`&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[[asyncIterator]](QueryBase.md#[asynciterator])
+[`QueryBase`](QueryBase.md).[`[asyncIterator]`](QueryBase.md#%5Basynciterator%5D)
 
-#### Defined in
+***
 
-[query.ts:154](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L154)
+### bypassVectorIndex()
 
-___
+> **bypassVectorIndex**(): [`VectorQuery`](VectorQuery.md)
-
-### bypassVectorIndex
-
-▸ **bypassVectorIndex**(): [`VectorQuery`](VectorQuery.md)
 
 If this is called then any vector index is skipped
 
@@ -113,15 +74,11 @@ calculate your recall to select an appropriate value for nprobes.
 
 [`VectorQuery`](VectorQuery.md)
 
-#### Defined in
+***
 
-[query.ts:321](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L321)
+### column()
 
-___
+> **column**(`column`): [`VectorQuery`](VectorQuery.md)
-
-### column
-
-▸ **column**(`column`): [`VectorQuery`](VectorQuery.md)
 
 Set the vector column to query
 
@@ -130,30 +87,24 @@ the call to
 
 #### Parameters
 
-| Name | Type |
+• **column**: `string`
-| :------ | :------ |
-| `column` | `string` |
 
 #### Returns
 
 [`VectorQuery`](VectorQuery.md)
 
-**`See`**
+#### See
 
 [Query#nearestTo](Query.md#nearestto)
 
 This parameter must be specified if the table has more than one column
 whose data type is a fixed-size-list of floats.
 
-#### Defined in
+***
 
-[query.ts:229](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L229)
+### distanceType()
 
-___
+> **distanceType**(`distanceType`): [`VectorQuery`](VectorQuery.md)
-
-### distanceType
-
-▸ **distanceType**(`distanceType`): [`VectorQuery`](VectorQuery.md)
 
 Set the distance metric to use
 
@@ -163,15 +114,13 @@ use.  See
 
 #### Parameters
 
-| Name | Type |
+• **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
-| :------ | :------ |
-| `distanceType` | `string` |
 
 #### Returns
 
 [`VectorQuery`](VectorQuery.md)
 
-**`See`**
+#### See
 
 [IvfPqOptions.distanceType](../interfaces/IvfPqOptions.md#distancetype) for more details on the different
 distance metrics available.
@@ -182,23 +131,41 @@ invalid.
 
 By default "l2" is used.
 
-#### Defined in
+***
 
-[query.ts:248](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L248)
+### doCall()
 
-___
+> `protected` **doCall**(`fn`): `void`
 
-### execute
+#### Parameters
 
-▸ **execute**(): [`RecordBatchIterator`](RecordBatchIterator.md)
+• **fn**
+
+#### Returns
+
+`void`
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`doCall`](QueryBase.md#docall)
+
+***
+
+### execute()
+
+> `protected` **execute**(`options`?): [`RecordBatchIterator`](RecordBatchIterator.md)
 
 Execute the query and return the results as an
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
 [`RecordBatchIterator`](RecordBatchIterator.md)
 
-**`See`**
+#### See
 
  - AsyncIterator
 of
@@ -212,17 +179,76 @@ single query)
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[execute](QueryBase.md#execute)
+[`QueryBase`](QueryBase.md).[`execute`](QueryBase.md#execute)
 
-#### Defined in
+***
 
-[query.ts:149](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L149)
+### explainPlan()
 
-___
+> **explainPlan**(`verbose`): `Promise`&lt;`string`&gt;
 
-### limit
+Generates an explanation of the query execution plan.
 
-▸ **limit**(`limit`): [`VectorQuery`](VectorQuery.md)
+#### Parameters
+
+• **verbose**: `boolean` = `false`
+
+If true, provides a more detailed explanation. Defaults to false.
+
+#### Returns
+
+`Promise`&lt;`string`&gt;
+
+A Promise that resolves to a string containing the query execution plan explanation.
+
+#### Example
+
+```ts
+import * as lancedb from "@lancedb/lancedb"
+const db = await lancedb.connect("./.lancedb");
+const table = await db.createTable("my_table", [
+  { vector: [1.1, 0.9], id: "1" },
+]);
+const plan = await table.query().nearestTo([0.5, 0.2]).explainPlan();
+```
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`explainPlan`](QueryBase.md#explainplan)
+
+***
+
+### ~~filter()~~
+
+> **filter**(`predicate`): `this`
+
+A filter statement to be applied to this query.
+
+#### Parameters
+
+• **predicate**: `string`
+
+#### Returns
+
+`this`
+
+#### Alias
+
+where
+
+#### Deprecated
+
+Use `where` instead
+
+#### Inherited from
+
+[`QueryBase`](QueryBase.md).[`filter`](QueryBase.md#filter)
+
+***
+
+### limit()
+
+> **limit**(`limit`): `this`
 
 Set the maximum number of results to return.
 
@@ -231,45 +257,39 @@ called then every valid row from the table will be returned.
 
 #### Parameters
 
-| Name | Type |
+• **limit**: `number`
-| :------ | :------ |
-| `limit` | `number` |
 
 #### Returns
 
-[`VectorQuery`](VectorQuery.md)
+`this`
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[limit](QueryBase.md#limit)
+[`QueryBase`](QueryBase.md).[`limit`](QueryBase.md#limit)
 
-#### Defined in
+***
 
-[query.ts:129](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L129)
+### nativeExecute()
 
-___
+> `protected` **nativeExecute**(`options`?): `Promise`&lt;`RecordBatchIterator`&gt;
 
-### nativeExecute
+#### Parameters
 
-▸ **nativeExecute**(): `Promise`\<`RecordBatchIterator`\>
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
 
 #### Returns
 
-`Promise`\<`RecordBatchIterator`\>
+`Promise`&lt;`RecordBatchIterator`&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[nativeExecute](QueryBase.md#nativeexecute)
+[`QueryBase`](QueryBase.md).[`nativeExecute`](QueryBase.md#nativeexecute)
 
-#### Defined in
+***
 
-[query.ts:134](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L134)
+### nprobes()
 
-___
+> **nprobes**(`nprobes`): [`VectorQuery`](VectorQuery.md)
-
-### nprobes
-
-▸ **nprobes**(`nprobes`): [`VectorQuery`](VectorQuery.md)
 
 Set the number of partitions to search (probe)
 
@@ -294,23 +314,17 @@ you the desired recall.
 
 #### Parameters
 
-| Name | Type |
+• **nprobes**: `number`
-| :------ | :------ |
-| `nprobes` | `number` |
 
 #### Returns
 
 [`VectorQuery`](VectorQuery.md)
 
-#### Defined in
+***
 
-[query.ts:215](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L215)
+### postfilter()
 
-___
+> **postfilter**(): [`VectorQuery`](VectorQuery.md)
-
-### postfilter
-
-▸ **postfilter**(): [`VectorQuery`](VectorQuery.md)
 
 If this is called then filtering will happen after the vector search instead of
 before.
@@ -333,20 +347,16 @@ Post filtering happens during the "refine stage" (described in more detail in
 
 [`VectorQuery`](VectorQuery.md)
 
-**`See`**
+#### See
 
 [VectorQuery#refineFactor](VectorQuery.md#refinefactor)).  This means that setting a higher refine
 factor can often help restore some of the results lost by post filtering.
 
-#### Defined in
+***
 
-[query.ts:307](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L307)
+### refineFactor()
 
-___
+> **refineFactor**(`refineFactor`): [`VectorQuery`](VectorQuery.md)
-
-### refineFactor
-
-▸ **refineFactor**(`refineFactor`): [`VectorQuery`](VectorQuery.md)
 
 A multiplier to control how many additional rows are taken during the refine step
 
@@ -378,23 +388,17 @@ distance between the query vector and the actual uncompressed vector.
 
 #### Parameters
 
-| Name | Type |
+• **refineFactor**: `number`
-| :------ | :------ |
-| `refineFactor` | `number` |
 
 #### Returns
 
 [`VectorQuery`](VectorQuery.md)
 
-#### Defined in
+***
 
-[query.ts:282](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L282)
+### select()
 
-___
+> **select**(`columns`): `this`
-
-### select
-
-▸ **select**(`columns`): [`VectorQuery`](VectorQuery.md)
 
 Return only the specified columns.
 
@@ -418,15 +422,13 @@ input to this method would be:
 
 #### Parameters
 
-| Name | Type |
+• **columns**: `string` \| `string`[] \| `Record`&lt;`string`, `string`&gt; \| `Map`&lt;`string`, `string`&gt;
-| :------ | :------ |
-| `columns` | `string`[] \| `Record`\<`string`, `string`\> \| `Map`\<`string`, `string`\> |
 
 #### Returns
 
-[`VectorQuery`](VectorQuery.md)
+`this`
 
-**`Example`**
+#### Example
 
 ```ts
 new Map([["combined", "a + b"], ["c", "c"]])
@@ -441,61 +443,57 @@ object insertion order is easy to get wrong and `Map` is more foolproof.
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[select](QueryBase.md#select)
+[`QueryBase`](QueryBase.md).[`select`](QueryBase.md#select)
 
-#### Defined in
+***
 
-[query.ts:108](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L108)
+### toArray()
 
-___
+> **toArray**(`options`?): `Promise`&lt;`any`[]&gt;
-
-### toArray
-
-▸ **toArray**(): `Promise`\<`unknown`[]\>
 
 Collect the results as an array of objects.
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
-`Promise`\<`unknown`[]\>
+`Promise`&lt;`any`[]&gt;
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[toArray](QueryBase.md#toarray)
+[`QueryBase`](QueryBase.md).[`toArray`](QueryBase.md#toarray)
 
-#### Defined in
+***
 
-[query.ts:169](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L169)
+### toArrow()
 
-___
+> **toArrow**(`options`?): `Promise`&lt;`Table`&lt;`any`&gt;&gt;
-
-### toArrow
-
-▸ **toArrow**(): `Promise`\<`Table`\<`any`\>\>
 
 Collect the results as an Arrow
 
+#### Parameters
+
+• **options?**: `Partial`&lt;`QueryExecutionOptions`&gt;
+
 #### Returns
 
-`Promise`\<`Table`\<`any`\>\>
+`Promise`&lt;`Table`&lt;`any`&gt;&gt;
 
-**`See`**
+#### See
 
 ArrowTable.
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[toArrow](QueryBase.md#toarrow)
+[`QueryBase`](QueryBase.md).[`toArrow`](QueryBase.md#toarrow)
 
-#### Defined in
+***
 
-[query.ts:160](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L160)
+### where()
 
-___
+> **where**(`predicate`): `this`
-
-### where
-
-▸ **where**(`predicate`): [`VectorQuery`](VectorQuery.md)
 
 A filter statement to be applied to this query.
 
@@ -503,15 +501,13 @@ The filter should be supplied as an SQL query string.  For example:
 
 #### Parameters
 
-| Name | Type |
+• **predicate**: `string`
-| :------ | :------ |
-| `predicate` | `string` |
 
 #### Returns
 
-[`VectorQuery`](VectorQuery.md)
+`this`
 
-**`Example`**
+#### Example
 
 ```ts
 x > 10
@@ -524,8 +520,4 @@ on the filter column(s).
 
 #### Inherited from
 
-[QueryBase](QueryBase.md).[where](QueryBase.md#where)
+[`QueryBase`](QueryBase.md).[`where`](QueryBase.md#where)
-
-#### Defined in
-
-[query.ts:73](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/query.ts#L73)

docs/src/js/classes/embedding.OpenAIEmbeddingFunction.md

@@ -1,111 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / [embedding](../modules/embedding.md) / OpenAIEmbeddingFunction
-
-# Class: OpenAIEmbeddingFunction
-
-[embedding](../modules/embedding.md).OpenAIEmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Implements
-
-- [`EmbeddingFunction`](../interfaces/embedding.EmbeddingFunction.md)\<`string`\>
-
-## Table of contents
-
-### Constructors
-
-- [constructor](embedding.OpenAIEmbeddingFunction.md#constructor)
-
-### Properties
-
-- [\_modelName](embedding.OpenAIEmbeddingFunction.md#_modelname)
-- [\_openai](embedding.OpenAIEmbeddingFunction.md#_openai)
-- [sourceColumn](embedding.OpenAIEmbeddingFunction.md#sourcecolumn)
-
-### Methods
-
-- [embed](embedding.OpenAIEmbeddingFunction.md#embed)
-
-## Constructors
-
-### constructor
-
-• **new OpenAIEmbeddingFunction**(`sourceColumn`, `openAIKey`, `modelName?`): [`OpenAIEmbeddingFunction`](embedding.OpenAIEmbeddingFunction.md)
-
-#### Parameters
-
-| Name | Type | Default value |
-| :------ | :------ | :------ |
-| `sourceColumn` | `string` | `undefined` |
-| `openAIKey` | `string` | `undefined` |
-| `modelName` | `string` | `"text-embedding-ada-002"` |
-
-#### Returns
-
-[`OpenAIEmbeddingFunction`](embedding.OpenAIEmbeddingFunction.md)
-
-#### Defined in
-
-[embedding/openai.ts:22](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L22)
-
-## Properties
-
-### \_modelName
-
-• `Private` `Readonly` **\_modelName**: `string`
-
-#### Defined in
-
-[embedding/openai.ts:20](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L20)
-
-___
-
-### \_openai
-
-• `Private` `Readonly` **\_openai**: `OpenAI`
-
-#### Defined in
-
-[embedding/openai.ts:19](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L19)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/embedding.EmbeddingFunction.md).[sourceColumn](../interfaces/embedding.EmbeddingFunction.md#sourcecolumn)
-
-#### Defined in
-
-[embedding/openai.ts:61](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L61)
-
-## Methods
-
-### embed
-
-▸ **embed**(`data`): `Promise`\<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `string`[] |
-
-#### Returns
-
-`Promise`\<`number`[][]\>
-
-#### Implementation of
-
-[EmbeddingFunction](../interfaces/embedding.EmbeddingFunction.md).[embed](../interfaces/embedding.EmbeddingFunction.md#embed)
-
-#### Defined in
-
-[embedding/openai.ts:48](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/openai.ts#L48)

docs/src/js/enumerations/WriteMode.md

@@ -0,0 +1,27 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / WriteMode
+
+# Enumeration: WriteMode
+
+Write mode for writing a table.
+
+## Enumeration Members
+
+### Append
+
+> **Append**: `"Append"`
+
+***
+
+### Create
+
+> **Create**: `"Create"`
+
+***
+
+### Overwrite
+
+> **Overwrite**: `"Overwrite"`

docs/src/js/enums/WriteMode.md

@@ -1,43 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / WriteMode
-
-# Enumeration: WriteMode
-
-Write mode for writing a table.
-
-## Table of contents
-
-### Enumeration Members
-
-- [Append](WriteMode.md#append)
-- [Create](WriteMode.md#create)
-- [Overwrite](WriteMode.md#overwrite)
-
-## Enumeration Members
-
-### Append
-
-• **Append** = ``"Append"``
-
-#### Defined in
-
-native.d.ts:69
-
-___
-
-### Create
-
-• **Create** = ``"Create"``
-
-#### Defined in
-
-native.d.ts:68
-
-___
-
-### Overwrite
-
-• **Overwrite** = ``"Overwrite"``
-
-#### Defined in
-
-native.d.ts:70

docs/src/js/functions/connect.md

@@ -0,0 +1,82 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / connect
+
+# Function: connect()
+
+## connect(uri, opts)
+
+> **connect**(`uri`, `opts`?): `Promise`<[`Connection`](../classes/Connection.md)>
+
+Connect to a LanceDB instance at the given URI.
+
+Accepted formats:
+
+- `/path/to/database` - local database
+- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+- `db://host:port` - remote database (LanceDB cloud)
+
+### Parameters
+
+• **uri**: `string`
+
+The uri of the database. If the database uri starts
+with `db://` then it connects to a remote database.
+
+• **opts?**: `Partial`<[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`>
+
+### Returns
+
+`Promise`<[`Connection`](../classes/Connection.md)>
+
+### See
+
+[ConnectionOptions](../interfaces/ConnectionOptions.md) for more details on the URI format.
+
+### Examples
+
+```ts
+const conn = await connect("/path/to/database");
+```
+
+```ts
+const conn = await connect(
+  "s3://bucket/path/to/database",
+  {storageOptions: {timeout: "60s"}
+});
+```
+
+## connect(opts)
+
+> **connect**(`opts`): `Promise`<[`Connection`](../classes/Connection.md)>
+
+Connect to a LanceDB instance at the given URI.
+
+Accepted formats:
+
+- `/path/to/database` - local database
+- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+- `db://host:port` - remote database (LanceDB cloud)
+
+### Parameters
+
+• **opts**: `Partial`<[`ConnectionOptions`](../interfaces/ConnectionOptions.md) \| `RemoteConnectionOptions`> & `object`
+
+### Returns
+
+`Promise`<[`Connection`](../classes/Connection.md)>
+
+### See
+
+[ConnectionOptions](../interfaces/ConnectionOptions.md) for more details on the URI format.
+
+### Example
+
+```ts
+const conn = await connect({
+  uri: "/path/to/database",
+  storageOptions: {timeout: "60s"}
+});
+```

docs/src/js/functions/makeArrowTable.md

@@ -1,103 +1,12 @@
-[@lancedb/lancedb](README.md) / Exports
+[**@lancedb/lancedb**](../README.md) • **Docs**
 
-# @lancedb/lancedb
+***
 
-## Table of contents
+[@lancedb/lancedb](../globals.md) / makeArrowTable
 
-### Namespaces
+# Function: makeArrowTable()
 
-- [embedding](modules/embedding.md)
+> **makeArrowTable**(`data`, `options`?, `metadata`?): `ArrowTable`
-
-### Enumerations
-
-- [WriteMode](enums/WriteMode.md)
-
-### Classes
-
-- [Connection](classes/Connection.md)
-- [Index](classes/Index.md)
-- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
-- [Query](classes/Query.md)
-- [QueryBase](classes/QueryBase.md)
-- [RecordBatchIterator](classes/RecordBatchIterator.md)
-- [Table](classes/Table.md)
-- [VectorColumnOptions](classes/VectorColumnOptions.md)
-- [VectorQuery](classes/VectorQuery.md)
-
-### Interfaces
-
-- [AddColumnsSql](interfaces/AddColumnsSql.md)
-- [AddDataOptions](interfaces/AddDataOptions.md)
-- [ColumnAlteration](interfaces/ColumnAlteration.md)
-- [ConnectionOptions](interfaces/ConnectionOptions.md)
-- [CreateTableOptions](interfaces/CreateTableOptions.md)
-- [ExecutableQuery](interfaces/ExecutableQuery.md)
-- [IndexConfig](interfaces/IndexConfig.md)
-- [IndexOptions](interfaces/IndexOptions.md)
-- [IvfPqOptions](interfaces/IvfPqOptions.md)
-- [TableNamesOptions](interfaces/TableNamesOptions.md)
-- [UpdateOptions](interfaces/UpdateOptions.md)
-- [WriteOptions](interfaces/WriteOptions.md)
-
-### Type Aliases
-
-- [Data](modules.md#data)
-
-### Functions
-
-- [connect](modules.md#connect)
-- [makeArrowTable](modules.md#makearrowtable)
-
-## Type Aliases
-
-### Data
-
-Ƭ **Data**: `Record`\<`string`, `unknown`\>[] \| `ArrowTable`
-
-Data type accepted by NodeJS SDK
-
-#### Defined in
-
-[arrow.ts:40](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L40)
-
-## Functions
-
-### connect
-
-▸ **connect**(`uri`, `opts?`): `Promise`\<[`Connection`](classes/Connection.md)\>
-
-Connect to a LanceDB instance at the given URI.
-
-Accpeted formats:
-
-- `/path/to/database` - local database
-- `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
-- `db://host:port` - remote database (LanceDB cloud)
-
-#### Parameters
-
-| Name | Type | Description |
-| :------ | :------ | :------ |
-| `uri` | `string` | The uri of the database. If the database uri starts with `db://` then it connects to a remote database. |
-| `opts?` | `Partial`\<[`ConnectionOptions`](interfaces/ConnectionOptions.md)\> | - |
-
-#### Returns
-
-`Promise`\<[`Connection`](classes/Connection.md)\>
-
-**`See`**
-
-[ConnectionOptions](interfaces/ConnectionOptions.md) for more details on the URI format.
-
-#### Defined in
-
-[index.ts:62](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/index.ts#L62)
-
-___
-
-### makeArrowTable
-
-▸ **makeArrowTable**(`data`, `options?`): `ArrowTable`
 
 An enhanced version of the makeTable function from Apache Arrow
 that supports nested fields and embeddings columns.
@@ -129,20 +38,20 @@ rules are as follows:
  - Record<String, any> => Struct
  - Array<any> => List
 
-#### Parameters
+## Parameters
 
-| Name | Type |
+• **data**: `Record`&lt;`string`, `unknown`&gt;[]
-| :------ | :------ |
-| `data` | `Record`\<`string`, `unknown`\>[] |
-| `options?` | `Partial`\<[`MakeArrowTableOptions`](classes/MakeArrowTableOptions.md)\> |
 
-#### Returns
+• **options?**: `Partial`&lt;[`MakeArrowTableOptions`](../classes/MakeArrowTableOptions.md)&gt;
+
+• **metadata?**: `Map`&lt;`string`, `string`&gt;
+
+## Returns
 
 `ArrowTable`
 
-**`Example`**
+## Example
 
-```ts
 import { fromTableToBuffer, makeArrowTable } from "../arrow";
 import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
 
@@ -203,7 +112,3 @@ const table = makeArrowTable([
  }
 assert.deepEqual(table.schema, schema)
 ```
-
-#### Defined in
-
-[arrow.ts:197](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/arrow.ts#L197)

docs/src/js/globals.md

@@ -0,0 +1,51 @@
+[**@lancedb/lancedb**](README.md) • **Docs**
+
+***
+
+# @lancedb/lancedb
+
+## Namespaces
+
+- [embedding](namespaces/embedding/README.md)
+
+## Enumerations
+
+- [WriteMode](enumerations/WriteMode.md)
+
+## Classes
+
+- [Connection](classes/Connection.md)
+- [Index](classes/Index.md)
+- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
+- [Query](classes/Query.md)
+- [QueryBase](classes/QueryBase.md)
+- [RecordBatchIterator](classes/RecordBatchIterator.md)
+- [Table](classes/Table.md)
+- [VectorColumnOptions](classes/VectorColumnOptions.md)
+- [VectorQuery](classes/VectorQuery.md)
+
+## Interfaces
+
+- [AddColumnsSql](interfaces/AddColumnsSql.md)
+- [AddDataOptions](interfaces/AddDataOptions.md)
+- [ColumnAlteration](interfaces/ColumnAlteration.md)
+- [ConnectionOptions](interfaces/ConnectionOptions.md)
+- [CreateTableOptions](interfaces/CreateTableOptions.md)
+- [ExecutableQuery](interfaces/ExecutableQuery.md)
+- [IndexConfig](interfaces/IndexConfig.md)
+- [IndexMetadata](interfaces/IndexMetadata.md)
+- [IndexOptions](interfaces/IndexOptions.md)
+- [IndexStatistics](interfaces/IndexStatistics.md)
+- [IvfPqOptions](interfaces/IvfPqOptions.md)
+- [TableNamesOptions](interfaces/TableNamesOptions.md)
+- [UpdateOptions](interfaces/UpdateOptions.md)
+- [WriteOptions](interfaces/WriteOptions.md)
+
+## Type Aliases
+
+- [Data](type-aliases/Data.md)
+
+## Functions
+
+- [connect](functions/connect.md)
+- [makeArrowTable](functions/makeArrowTable.md)

docs/src/js/interfaces/AddColumnsSql.md

@@ -1,37 +1,26 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / AddColumnsSql
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AddColumnsSql
 
 # Interface: AddColumnsSql
 
 A definition of a new column to add to a table.
 
-## Table of contents
-
-### Properties
-
-- [name](AddColumnsSql.md#name)
-- [valueSql](AddColumnsSql.md#valuesql)
-
 ## Properties
 
 ### name
 
-• **name**: `string`
+> **name**: `string`
 
 The name of the new column.
 
-#### Defined in
+***
-
-native.d.ts:43
-
-___
 
 ### valueSql
 
-• **valueSql**: `string`
+> **valueSql**: `string`
 
 The values to populate the new column with, as a SQL expression.
 The expression can reference other columns in the table.
-
-#### Defined in
-
-native.d.ts:48

docs/src/js/interfaces/AddDataOptions.md

@@ -1,25 +1,19 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / AddDataOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / AddDataOptions
 
 # Interface: AddDataOptions
 
 Options for adding data to a table.
 
-## Table of contents
-
-### Properties
-
-- [mode](AddDataOptions.md#mode)
-
 ## Properties
 
 ### mode
 
-• **mode**: ``"append"`` \| ``"overwrite"``
+> **mode**: `"append"` \| `"overwrite"`
 
 If "append" (the default) then the new data will be added to the table
 
 If "overwrite" then the new data will replace the existing data in the table.
-
-#### Defined in
-
-[table.ts:36](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L36)

docs/src/js/interfaces/ColumnAlteration.md

@@ -1,4 +1,8 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / ColumnAlteration
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ColumnAlteration
 
 # Interface: ColumnAlteration
 
@@ -7,50 +11,30 @@ A definition of a column alteration. The alteration changes the column at
 and to have the data type `data_type`. At least one of `rename` or `nullable`
 must be provided.
 
-## Table of contents
-
-### Properties
-
-- [nullable](ColumnAlteration.md#nullable)
-- [path](ColumnAlteration.md#path)
-- [rename](ColumnAlteration.md#rename)
-
 ## Properties
 
-### nullable
+### nullable?
 
-• `Optional` **nullable**: `boolean`
+> `optional` **nullable**: `boolean`
 
 Set the new nullability. Note that a nullable column cannot be made non-nullable.
 
-#### Defined in
+***
-
-native.d.ts:38
-
-___
 
 ### path
 
-• **path**: `string`
+> **path**: `string`
 
 The path to the column to alter. This is a dot-separated path to the column.
 If it is a top-level column then it is just the name of the column. If it is
 a nested column then it is the path to the column, e.g. "a.b.c" for a column
 `c` nested inside a column `b` nested inside a column `a`.
 
-#### Defined in
+***
 
-native.d.ts:31
+### rename?
 
-___
+> `optional` **rename**: `string`
-
-### rename
-
-• `Optional` **rename**: `string`
 
 The new name of the column. If not provided then the name will not be changed.
 This must be distinct from the names of all other columns in the table.
-
-#### Defined in
-
-native.d.ts:36

docs/src/js/interfaces/ConnectionOptions.md

@@ -1,40 +1,16 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / ConnectionOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ConnectionOptions
 
 # Interface: ConnectionOptions
 
-## Table of contents
-
-### Properties
-
-- [apiKey](ConnectionOptions.md#apikey)
-- [hostOverride](ConnectionOptions.md#hostoverride)
-- [readConsistencyInterval](ConnectionOptions.md#readconsistencyinterval)
-
 ## Properties
 
-### apiKey
+### readConsistencyInterval?
 
-• `Optional` **apiKey**: `string`
+> `optional` **readConsistencyInterval**: `number`
-
-#### Defined in
-
-native.d.ts:51
-
-___
-
-### hostOverride
-
-• `Optional` **hostOverride**: `string`
-
-#### Defined in
-
-native.d.ts:52
-
-___
-
-### readConsistencyInterval
-
-• `Optional` **readConsistencyInterval**: `number`
 
 (For LanceDB OSS only): The interval, in seconds, at which to check for
 updates to the table from other processes. If None, then consistency is not
@@ -46,6 +22,12 @@ has passed since the last check, then the table will be checked for updates.
 Note: this consistency only applies to read operations. Write operations are
 always consistent.
 
-#### Defined in
+***
 
-native.d.ts:64
+### storageOptions?
+
+> `optional` **storageOptions**: `Record`<`string`, `string`>
+
+(For LanceDB OSS only): configuration for object storage.
+
+The available options are described at https://lancedb.github.io/lancedb/guides/storage/

docs/src/js/interfaces/CreateTableOptions.md

@@ -1,32 +1,31 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / CreateTableOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / CreateTableOptions
 
 # Interface: CreateTableOptions
 
-## Table of contents
-
-### Properties
-
-- [existOk](CreateTableOptions.md#existok)
-- [mode](CreateTableOptions.md#mode)
-
 ## Properties
 
+### embeddingFunction?
+
+> `optional` **embeddingFunction**: [`EmbeddingFunctionConfig`](../namespaces/embedding/interfaces/EmbeddingFunctionConfig.md)
+
+***
+
 ### existOk
 
-• **existOk**: `boolean`
+> **existOk**: `boolean`
 
 If this is true and the table already exists and the mode is "create"
 then no error will be raised.
 
-#### Defined in
+***
-
-[connection.ts:35](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L35)
-
-___
 
 ### mode
 
-• **mode**: ``"overwrite"`` \| ``"create"``
+> **mode**: `"overwrite"` \| `"create"`
 
 The mode to use when creating the table.
 
@@ -36,6 +35,31 @@ happen.  Any provided data will be ignored.
 
 If this is set to "overwrite" then any existing table will be replaced.
 
-#### Defined in
+***
 
-[connection.ts:30](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L30)
+### schema?
+
+> `optional` **schema**: `SchemaLike`
+
+***
+
+### storageOptions?
+
+> `optional` **storageOptions**: `Record`<`string`, `string`>
+
+Configuration for object storage.
+
+Options already set on the connection will be inherited by the table,
+but can be overridden here.
+
+The available options are described at https://lancedb.github.io/lancedb/guides/storage/
+
+***
+
+### useLegacyFormat?
+
+> `optional` **useLegacyFormat**: `boolean`
+
+If true then data files will be written with the legacy format
+
+The default is true while the new format is in beta

docs/src/js/interfaces/ExecutableQuery.md

@@ -1,4 +1,8 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / ExecutableQuery
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / ExecutableQuery
 
 # Interface: ExecutableQuery
 

docs/src/js/interfaces/IndexConfig.md

@@ -1,39 +1,36 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / IndexConfig
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexConfig
 
 # Interface: IndexConfig
 
 A description of an index currently configured on a column
 
-## Table of contents
-
-### Properties
-
-- [columns](IndexConfig.md#columns)
-- [indexType](IndexConfig.md#indextype)
-
 ## Properties
 
 ### columns
 
-• **columns**: `string`[]
+> **columns**: `string`[]
 
 The columns in the index
 
-Currently this is always an array of size 1.  In the future there may
+Currently this is always an array of size 1. In the future there may
 be more columns to represent composite indices.
 
-#### Defined in
+***
-
-native.d.ts:16
-
-___
 
 ### indexType
 
-• **indexType**: `string`
+> **indexType**: `string`
 
 The type of the index
 
-#### Defined in
+***
 
-native.d.ts:9
+### name
+
+> **name**: `string`
+
+The name of the index

docs/src/js/interfaces/IndexMetadata.md

@@ -0,0 +1,19 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexMetadata
+
+# Interface: IndexMetadata
+
+## Properties
+
+### indexType?
+
+> `optional` **indexType**: `string`
+
+***
+
+### metricType?
+
+> `optional` **metricType**: `string`

docs/src/js/interfaces/IndexOptions.md

@@ -1,19 +1,16 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / IndexOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexOptions
 
 # Interface: IndexOptions
 
-## Table of contents
-
-### Properties
-
-- [config](IndexOptions.md#config)
-- [replace](IndexOptions.md#replace)
-
 ## Properties
 
-### config
+### config?
 
-• `Optional` **config**: [`Index`](../classes/Index.md)
+> `optional` **config**: [`Index`](../classes/Index.md)
 
 Advanced index configuration
 
@@ -25,15 +22,11 @@ See the static methods on Index for details on the various index types.
 If this is not supplied then column data type(s) and column statistics
 will be used to determine the most useful kind of index to create.
 
-#### Defined in
+***
 
-[indices.ts:192](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L192)
+### replace?
 
-___
+> `optional` **replace**: `boolean`
-
-### replace
-
-• `Optional` **replace**: `boolean`
 
 Whether to replace the existing index
 
@@ -42,7 +35,3 @@ and the same name, then an error will be returned.  This is true even if
 that index is out of date.
 
 The default is true
-
-#### Defined in
-
-[indices.ts:202](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L202)

docs/src/js/interfaces/IndexStatistics.md

@@ -0,0 +1,39 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IndexStatistics
+
+# Interface: IndexStatistics
+
+## Properties
+
+### indexType?
+
+> `optional` **indexType**: `string`
+
+The type of the index
+
+***
+
+### indices
+
+> **indices**: [`IndexMetadata`](IndexMetadata.md)[]
+
+The metadata for each index
+
+***
+
+### numIndexedRows
+
+> **numIndexedRows**: `number`
+
+The number of rows indexed by the index
+
+***
+
+### numUnindexedRows
+
+> **numUnindexedRows**: `number`
+
+The number of rows not indexed

docs/src/js/interfaces/IvfPqOptions.md

@@ -1,24 +1,18 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / IvfPqOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / IvfPqOptions
 
 # Interface: IvfPqOptions
 
 Options to create an `IVF_PQ` index
 
-## Table of contents
-
-### Properties
-
-- [distanceType](IvfPqOptions.md#distancetype)
-- [maxIterations](IvfPqOptions.md#maxiterations)
-- [numPartitions](IvfPqOptions.md#numpartitions)
-- [numSubVectors](IvfPqOptions.md#numsubvectors)
-- [sampleRate](IvfPqOptions.md#samplerate)
-
 ## Properties
 
-### distanceType
+### distanceType?
 
-• `Optional` **distanceType**: ``"l2"`` \| ``"cosine"`` \| ``"dot"``
+> `optional` **distanceType**: `"l2"` \| `"cosine"` \| `"dot"`
 
 Distance type to use to build the index.
 
@@ -52,15 +46,11 @@ never be returned from a vector search.
 distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
 L2 norm is 1), then dot distance is equivalent to the cosine distance.
 
-#### Defined in
+***
 
-[indices.ts:83](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L83)
+### maxIterations?
 
-___
+> `optional` **maxIterations**: `number`
-
-### maxIterations
-
-• `Optional` **maxIterations**: `number`
 
 Max iteration to train IVF kmeans.
 
@@ -72,15 +62,11 @@ iterations have diminishing returns.
 
 The default value is 50.
 
-#### Defined in
+***
 
-[indices.ts:96](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L96)
+### numPartitions?
 
-___
+> `optional` **numPartitions**: `number`
-
-### numPartitions
-
-• `Optional` **numPartitions**: `number`
 
 The number of IVF partitions to create.
 
@@ -92,15 +78,11 @@ If this value is too large then the first part of the search (picking the
 right partition) will be slow.  If this value is too small then the second
 part of the search (searching within a partition) will be slow.
 
-#### Defined in
+***
 
-[indices.ts:32](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L32)
+### numSubVectors?
 
-___
+> `optional` **numSubVectors**: `number`
-
-### numSubVectors
-
-• `Optional` **numSubVectors**: `number`
 
 Number of sub-vectors of PQ.
 
@@ -115,15 +97,11 @@ us to use efficient SIMD instructions.
 If the dimension is not visible by 8 then we use 1 subvector.  This is not ideal and
 will likely result in poor performance.
 
-#### Defined in
+***
 
-[indices.ts:48](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L48)
+### sampleRate?
 
-___
+> `optional` **sampleRate**: `number`
-
-### sampleRate
-
-• `Optional` **sampleRate**: `number`
 
 The number of vectors, per partition, to sample when training IVF kmeans.
 
@@ -138,7 +116,3 @@ Increasing this value might improve the quality of the index but in most cases t
 default should be sufficient.
 
 The default value is 256.
-
-#### Defined in
-
-[indices.ts:113](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/indices.ts#L113)

docs/src/js/interfaces/TableNamesOptions.md

@@ -1,38 +1,27 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / TableNamesOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / TableNamesOptions
 
 # Interface: TableNamesOptions
 
-## Table of contents
-
-### Properties
-
-- [limit](TableNamesOptions.md#limit)
-- [startAfter](TableNamesOptions.md#startafter)
-
 ## Properties
 
-### limit
+### limit?
 
-• `Optional` **limit**: `number`
+> `optional` **limit**: `number`
 
 An optional limit to the number of results to return.
 
-#### Defined in
+***
 
-[connection.ts:48](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L48)
+### startAfter?
 
-___
+> `optional` **startAfter**: `string`
-
-### startAfter
-
-• `Optional` **startAfter**: `string`
 
 If present, only return names that come lexicographically after the
 supplied value.
 
 This can be combined with limit to implement pagination by setting this to
 the last table name from the previous page.
-
-#### Defined in
-
-[connection.ts:46](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/connection.ts#L46)

docs/src/js/interfaces/UpdateOptions.md

@@ -1,18 +1,16 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / UpdateOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / UpdateOptions
 
 # Interface: UpdateOptions
 
-## Table of contents
-
-### Properties
-
-- [where](UpdateOptions.md#where)
-
 ## Properties
 
 ### where
 
-• **where**: `string`
+> **where**: `string`
 
 A filter that limits the scope of the update.
 
@@ -22,7 +20,3 @@ Only rows that satisfy the expression will be updated.
 
 For example, this could be 'my_col == 0' to replace all instances
 of 0 in a column with some other default value.
-
-#### Defined in
-
-[table.ts:50](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/table.ts#L50)

docs/src/js/interfaces/WriteOptions.md

@@ -1,21 +1,17 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / WriteOptions
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / WriteOptions
 
 # Interface: WriteOptions
 
 Write options when creating a Table.
 
-## Table of contents
-
-### Properties
-
-- [mode](WriteOptions.md#mode)
-
 ## Properties
 
-### mode
+### mode?
 
-• `Optional` **mode**: [`WriteMode`](../enums/WriteMode.md)
+> `optional` **mode**: [`WriteMode`](../enumerations/WriteMode.md)
 
-#### Defined in
+Write mode for writing to a table.
-
-native.d.ts:74

docs/src/js/interfaces/embedding.EmbeddingFunction.md

@@ -1,129 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / [embedding](../modules/embedding.md) / EmbeddingFunction
-
-# Interface: EmbeddingFunction\<T\>
-
-[embedding](../modules/embedding.md).EmbeddingFunction
-
-An embedding function that automatically creates vector representation for a given column.
-
-## Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-## Implemented by
-
-- [`OpenAIEmbeddingFunction`](../classes/embedding.OpenAIEmbeddingFunction.md)
-
-## Table of contents
-
-### Properties
-
-- [destColumn](embedding.EmbeddingFunction.md#destcolumn)
-- [embed](embedding.EmbeddingFunction.md#embed)
-- [embeddingDataType](embedding.EmbeddingFunction.md#embeddingdatatype)
-- [embeddingDimension](embedding.EmbeddingFunction.md#embeddingdimension)
-- [excludeSource](embedding.EmbeddingFunction.md#excludesource)
-- [sourceColumn](embedding.EmbeddingFunction.md#sourcecolumn)
-
-## Properties
-
-### destColumn
-
-• `Optional` **destColumn**: `string`
-
-The name of the column that will contain the embedding
-
-By default this is "vector"
-
-#### Defined in
-
-[embedding/embedding_function.ts:49](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L49)
-
-___
-
-### embed
-
-• **embed**: (`data`: `T`[]) => `Promise`\<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-#### Type declaration
-
-▸ (`data`): `Promise`\<`number`[][]\>
-
-Creates a vector representation for the given values.
-
-##### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `data` | `T`[] |
-
-##### Returns
-
-`Promise`\<`number`[][]\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:62](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L62)
-
-___
-
-### embeddingDataType
-
-• `Optional` **embeddingDataType**: `Float`\<`Floats`\>
-
-The data type of the embedding
-
-The embedding function should return `number`.  This will be converted into
-an Arrow float array.  By default this will be Float32 but this property can
-be used to control the conversion.
-
-#### Defined in
-
-[embedding/embedding_function.ts:33](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L33)
-
-___
-
-### embeddingDimension
-
-• `Optional` **embeddingDimension**: `number`
-
-The dimension of the embedding
-
-This is optional, normally this can be determined by looking at the results of
-`embed`.  If this is not specified, and there is an attempt to apply the embedding
-to an empty table, then that process will fail.
-
-#### Defined in
-
-[embedding/embedding_function.ts:42](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L42)
-
-___
-
-### excludeSource
-
-• `Optional` **excludeSource**: `boolean`
-
-Should the source column be excluded from the resulting table
-
-By default the source column is included.  Set this to true and
-only the embedding will be stored.
-
-#### Defined in
-
-[embedding/embedding_function.ts:57](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L57)
-
-___
-
-### sourceColumn
-
-• **sourceColumn**: `string`
-
-The name of the column that will be used as input for the Embedding Function.
-
-#### Defined in
-
-[embedding/embedding_function.ts:24](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L24)

docs/src/js/modules/embedding.md

@@ -1,45 +0,0 @@
-[@lancedb/lancedb](../README.md) / [Exports](../modules.md) / embedding
-
-# Namespace: embedding
-
-## Table of contents
-
-### Classes
-
-- [OpenAIEmbeddingFunction](../classes/embedding.OpenAIEmbeddingFunction.md)
-
-### Interfaces
-
-- [EmbeddingFunction](../interfaces/embedding.EmbeddingFunction.md)
-
-### Functions
-
-- [isEmbeddingFunction](embedding.md#isembeddingfunction)
-
-## Functions
-
-### isEmbeddingFunction
-
-▸ **isEmbeddingFunction**\<`T`\>(`value`): value is EmbeddingFunction\<T\>
-
-Test if the input seems to be an embedding function
-
-#### Type parameters
-
-| Name |
-| :------ |
-| `T` |
-
-#### Parameters
-
-| Name | Type |
-| :------ | :------ |
-| `value` | `unknown` |
-
-#### Returns
-
-value is EmbeddingFunction\<T\>
-
-#### Defined in
-
-[embedding/embedding_function.ts:66](https://github.com/lancedb/lancedb/blob/9d178c7/nodejs/lancedb/embedding/embedding_function.ts#L66)

docs/src/js/namespaces/embedding/README.md

@@ -0,0 +1,29 @@
+[**@lancedb/lancedb**](../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../globals.md) / embedding
+
+# embedding
+
+## Index
+
+### Classes
+
+- [EmbeddingFunction](classes/EmbeddingFunction.md)
+- [EmbeddingFunctionRegistry](classes/EmbeddingFunctionRegistry.md)
+- [OpenAIEmbeddingFunction](classes/OpenAIEmbeddingFunction.md)
+
+### Interfaces
+
+- [EmbeddingFunctionConfig](interfaces/EmbeddingFunctionConfig.md)
+
+### Type Aliases
+
+- [OpenAIOptions](type-aliases/OpenAIOptions.md)
+
+### Functions
+
+- [LanceSchema](functions/LanceSchema.md)
+- [getRegistry](functions/getRegistry.md)
+- [register](functions/register.md)

docs/src/js/namespaces/embedding/classes/EmbeddingFunction.md

@@ -0,0 +1,162 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / EmbeddingFunction
+
+# Class: `abstract` EmbeddingFunction<T, M>
+
+An embedding function that automatically creates vector representation for a given column.
+
+## Extended by
+
+- [`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+
+## Type Parameters
+
+• **T** = `any`
+
+• **M** *extends* `FunctionOptions` = `FunctionOptions`
+
+## Constructors
+
+### new EmbeddingFunction()
+
+> **new EmbeddingFunction**<`T`, `M`>(): [`EmbeddingFunction`](EmbeddingFunction.md)<`T`, `M`>
+
+#### Returns
+
+[`EmbeddingFunction`](EmbeddingFunction.md)<`T`, `M`>
+
+## Methods
+
+### computeQueryEmbeddings()
+
+> **computeQueryEmbeddings**(`data`): `Promise`<`number`[] \| `Float32Array` \| `Float64Array`>
+
+Compute the embeddings for a single query
+
+#### Parameters
+
+• **data**: `T`
+
+#### Returns
+
+`Promise`<`number`[] \| `Float32Array` \| `Float64Array`>
+
+***
+
+### computeSourceEmbeddings()
+
+> `abstract` **computeSourceEmbeddings**(`data`): `Promise`<`number`[][] \| `Float32Array`[] \| `Float64Array`[]>
+
+Creates a vector representation for the given values.
+
+#### Parameters
+
+• **data**: `T`[]
+
+#### Returns
+
+`Promise`<`number`[][] \| `Float32Array`[] \| `Float64Array`[]>
+
+***
+
+### embeddingDataType()
+
+> `abstract` **embeddingDataType**(): `Float`<`Floats`>
+
+The datatype of the embeddings
+
+#### Returns
+
+`Float`<`Floats`>
+
+***
+
+### ndims()
+
+> **ndims**(): `undefined` \| `number`
+
+The number of dimensions of the embeddings
+
+#### Returns
+
+`undefined` \| `number`
+
+***
+
+### sourceField()
+
+> **sourceField**(`optionsOrDatatype`): [`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+sourceField is used in combination with `LanceSchema` to provide a declarative data model
+
+#### Parameters
+
+• **optionsOrDatatype**: `DataType`<`Type`, `any`> \| `Partial`<`FieldOptions`<`DataType`<`Type`, `any`>>>
+
+The options for the field or the datatype
+
+#### Returns
+
+[`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+#### See
+
+lancedb.LanceSchema
+
+***
+
+### toJSON()
+
+> `abstract` **toJSON**(): `Partial`<`M`>
+
+Convert the embedding function to a JSON object
+It is used to serialize the embedding function to the schema
+It's important that any object returned by this method contains all the necessary
+information to recreate the embedding function
+
+It should return the same object that was passed to the constructor
+If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
+
+#### Returns
+
+`Partial`<`M`>
+
+#### Example
+
+```ts
+class MyEmbeddingFunction extends EmbeddingFunction {
+  constructor(options: {model: string, timeout: number}) {
+    super();
+    this.model = options.model;
+    this.timeout = options.timeout;
+  }
+  toJSON() {
+    return {
+      model: this.model,
+      timeout: this.timeout,
+    };
+}
+```
+
+***
+
+### vectorField()
+
+> **vectorField**(`optionsOrDatatype`?): [`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+vectorField is used in combination with `LanceSchema` to provide a declarative data model
+
+#### Parameters
+
+• **optionsOrDatatype?**: `DataType`<`Type`, `any`> \| `Partial`<`FieldOptions`<`DataType`<`Type`, `any`>>>
+
+#### Returns
+
+[`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+#### See
+
+lancedb.LanceSchema

docs/src/js/namespaces/embedding/classes/EmbeddingFunctionRegistry.md

@@ -0,0 +1,124 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / EmbeddingFunctionRegistry
+
+# Class: EmbeddingFunctionRegistry
+
+This is a singleton class used to register embedding functions
+and fetch them by name. It also handles serializing and deserializing.
+You can implement your own embedding function by subclassing EmbeddingFunction
+or TextEmbeddingFunction and registering it with the registry
+
+## Constructors
+
+### new EmbeddingFunctionRegistry()
+
+> **new EmbeddingFunctionRegistry**(): [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+
+#### Returns
+
+[`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+
+## Methods
+
+### functionToMetadata()
+
+> **functionToMetadata**(`conf`): `Record`<`string`, `any`>
+
+#### Parameters
+
+• **conf**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)
+
+#### Returns
+
+`Record`<`string`, `any`>
+
+***
+
+### get()
+
+> **get**<`T`, `Name`>(`name`): `Name` *extends* `"openai"` ? `EmbeddingFunctionCreate`<[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)> : `undefined` \| `EmbeddingFunctionCreate`<`T`>
+
+Fetch an embedding function by name
+
+#### Type Parameters
+
+• **T** *extends* [`EmbeddingFunction`](EmbeddingFunction.md)<`unknown`, `FunctionOptions`>
+
+• **Name** *extends* `string` = `""`
+
+#### Parameters
+
+• **name**: `Name` *extends* `"openai"` ? `"openai"` : `string`
+
+The name of the function
+
+#### Returns
+
+`Name` *extends* `"openai"` ? `EmbeddingFunctionCreate`<[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)> : `undefined` \| `EmbeddingFunctionCreate`<`T`>
+
+***
+
+### getTableMetadata()
+
+> **getTableMetadata**(`functions`): `Map`<`string`, `string`>
+
+#### Parameters
+
+• **functions**: [`EmbeddingFunctionConfig`](../interfaces/EmbeddingFunctionConfig.md)[]
+
+#### Returns
+
+`Map`<`string`, `string`>
+
+***
+
+### register()
+
+> **register**<`T`>(`this`, `alias`?): (`ctor`) => `any`
+
+Register an embedding function
+
+#### Type Parameters
+
+• **T** *extends* `EmbeddingFunctionConstructor`<[`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>> = `EmbeddingFunctionConstructor`<[`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>
+
+#### Parameters
+
+• **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+
+• **alias?**: `string`
+
+#### Returns
+
+`Function`
+
+##### Parameters
+
+• **ctor**: `T`
+
+##### Returns
+
+`any`
+
+#### Throws
+
+Error if the function is already registered
+
+***
+
+### reset()
+
+> **reset**(`this`): `void`
+
+reset the registry to the initial state
+
+#### Parameters
+
+• **this**: [`EmbeddingFunctionRegistry`](EmbeddingFunctionRegistry.md)
+
+#### Returns
+
+`void`

docs/src/js/namespaces/embedding/classes/OpenAIEmbeddingFunction.md

@@ -0,0 +1,196 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / OpenAIEmbeddingFunction
+
+# Class: OpenAIEmbeddingFunction
+
+An embedding function that automatically creates vector representation for a given column.
+
+## Extends
+
+- [`EmbeddingFunction`](EmbeddingFunction.md)<`string`, `Partial`<[`OpenAIOptions`](../type-aliases/OpenAIOptions.md)>>
+
+## Constructors
+
+### new OpenAIEmbeddingFunction()
+
+> **new OpenAIEmbeddingFunction**(`options`): [`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+
+#### Parameters
+
+• **options**: `Partial`<[`OpenAIOptions`](../type-aliases/OpenAIOptions.md)> = `...`
+
+#### Returns
+
+[`OpenAIEmbeddingFunction`](OpenAIEmbeddingFunction.md)
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`constructor`](EmbeddingFunction.md#constructors)
+
+## Methods
+
+### computeQueryEmbeddings()
+
+> **computeQueryEmbeddings**(`data`): `Promise`<`number`[]>
+
+Compute the embeddings for a single query
+
+#### Parameters
+
+• **data**: `string`
+
+#### Returns
+
+`Promise`<`number`[]>
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`computeQueryEmbeddings`](EmbeddingFunction.md#computequeryembeddings)
+
+***
+
+### computeSourceEmbeddings()
+
+> **computeSourceEmbeddings**(`data`): `Promise`<`number`[][]>
+
+Creates a vector representation for the given values.
+
+#### Parameters
+
+• **data**: `string`[]
+
+#### Returns
+
+`Promise`<`number`[][]>
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`computeSourceEmbeddings`](EmbeddingFunction.md#computesourceembeddings)
+
+***
+
+### embeddingDataType()
+
+> **embeddingDataType**(): `Float`<`Floats`>
+
+The datatype of the embeddings
+
+#### Returns
+
+`Float`<`Floats`>
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`embeddingDataType`](EmbeddingFunction.md#embeddingdatatype)
+
+***
+
+### ndims()
+
+> **ndims**(): `number`
+
+The number of dimensions of the embeddings
+
+#### Returns
+
+`number`
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`ndims`](EmbeddingFunction.md#ndims)
+
+***
+
+### sourceField()
+
+> **sourceField**(`optionsOrDatatype`): [`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+sourceField is used in combination with `LanceSchema` to provide a declarative data model
+
+#### Parameters
+
+• **optionsOrDatatype**: `DataType`<`Type`, `any`> \| `Partial`<`FieldOptions`<`DataType`<`Type`, `any`>>>
+
+The options for the field or the datatype
+
+#### Returns
+
+[`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+#### See
+
+lancedb.LanceSchema
+
+#### Inherited from
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`sourceField`](EmbeddingFunction.md#sourcefield)
+
+***
+
+### toJSON()
+
+> **toJSON**(): `object`
+
+Convert the embedding function to a JSON object
+It is used to serialize the embedding function to the schema
+It's important that any object returned by this method contains all the necessary
+information to recreate the embedding function
+
+It should return the same object that was passed to the constructor
+If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
+
+#### Returns
+
+`object`
+
+##### model
+
+> **model**: `string` & `object` \| `"text-embedding-ada-002"` \| `"text-embedding-3-small"` \| `"text-embedding-3-large"`
+
+#### Example
+
+```ts
+class MyEmbeddingFunction extends EmbeddingFunction {
+  constructor(options: {model: string, timeout: number}) {
+    super();
+    this.model = options.model;
+    this.timeout = options.timeout;
+  }
+  toJSON() {
+    return {
+      model: this.model,
+      timeout: this.timeout,
+    };
+}
+```
+
+#### Overrides
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`toJSON`](EmbeddingFunction.md#tojson)
+
+***
+
+### vectorField()
+
+> **vectorField**(`optionsOrDatatype`?): [`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+vectorField is used in combination with `LanceSchema` to provide a declarative data model
+
+#### Parameters
+
+• **optionsOrDatatype?**: `DataType`<`Type`, `any`> \| `Partial`<`FieldOptions`<`DataType`<`Type`, `any`>>>
+
+#### Returns
+
+[`DataType`<`Type`, `any`>, `Map`<`string`, [`EmbeddingFunction`](EmbeddingFunction.md)<`any`, `FunctionOptions`>>]
+
+#### See
+
+lancedb.LanceSchema
+
+#### Inherited from
+
+[`EmbeddingFunction`](EmbeddingFunction.md).[`vectorField`](EmbeddingFunction.md#vectorfield)

docs/src/js/namespaces/embedding/functions/LanceSchema.md

@@ -0,0 +1,39 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / LanceSchema
+
+# Function: LanceSchema()
+
+> **LanceSchema**(`fields`): `Schema`
+
+Create a schema with embedding functions.
+
+## Parameters
+
+• **fields**: `Record`<`string`, `object` \| [`object`, `Map`<`string`, [`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>>]>
+
+## Returns
+
+`Schema`
+
+Schema
+
+## Example
+
+```ts
+class MyEmbeddingFunction extends EmbeddingFunction {
+// ...
+}
+const func = new MyEmbeddingFunction();
+const schema = LanceSchema({
+  id: new Int32(),
+  text: func.sourceField(new Utf8()),
+  vector: func.vectorField(),
+  // optional: specify the datatype and/or dimensions
+  vector2: func.vectorField({ datatype: new Float32(), dims: 3}),
+});
+
+const table = await db.createTable("my_table", data, { schema });
+```

docs/src/js/namespaces/embedding/functions/getRegistry.md

@@ -0,0 +1,23 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / getRegistry
+
+# Function: getRegistry()
+
+> **getRegistry**(): [`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
+
+Utility function to get the global instance of the registry
+
+## Returns
+
+[`EmbeddingFunctionRegistry`](../classes/EmbeddingFunctionRegistry.md)
+
+`EmbeddingFunctionRegistry` The global instance of the registry
+
+## Example
+
+```ts
+const registry = getRegistry();
+const openai = registry.get("openai").create();

docs/src/js/namespaces/embedding/functions/register.md

@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / register
+
+# Function: register()
+
+> **register**(`name`?): (`ctor`) => `any`
+
+## Parameters
+
+• **name?**: `string`
+
+## Returns
+
+`Function`
+
+### Parameters
+
+• **ctor**: `EmbeddingFunctionConstructor`<[`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>>
+
+### Returns
+
+`any`

docs/src/js/namespaces/embedding/interfaces/EmbeddingFunctionConfig.md

@@ -0,0 +1,25 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / EmbeddingFunctionConfig
+
+# Interface: EmbeddingFunctionConfig
+
+## Properties
+
+### function
+
+> **function**: [`EmbeddingFunction`](../classes/EmbeddingFunction.md)<`any`, `FunctionOptions`>
+
+***
+
+### sourceColumn
+
+> **sourceColumn**: `string`
+
+***
+
+### vectorColumn?
+
+> `optional` **vectorColumn**: `string`

docs/src/js/namespaces/embedding/type-aliases/OpenAIOptions.md

@@ -0,0 +1,19 @@
+[**@lancedb/lancedb**](../../../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../../../globals.md) / [embedding](../README.md) / OpenAIOptions
+
+# Type Alias: OpenAIOptions
+
+> **OpenAIOptions**: `object`
+
+## Type declaration
+
+### apiKey
+
+> **apiKey**: `string`
+
+### model
+
+> **model**: `EmbeddingCreateParams`\[`"model"`\]

docs/src/js/type-aliases/Data.md

@@ -0,0 +1,11 @@
+[**@lancedb/lancedb**](../README.md) • **Docs**
+
+***
+
+[@lancedb/lancedb](../globals.md) / Data
+
+# Type Alias: Data
+
+> **Data**: `Record`<`string`, `unknown`>[] \| `TableLike`
+
+Data type accepted by NodeJS SDK

docs/src/migration.md

@@ -9,7 +9,8 @@ around the asynchronous client.
 This guide describes the differences between the two APIs and will hopefully assist users
 that would like to migrate to the new API.
 
-## Closeable Connections
+## Python
+### Closeable Connections
 
 The Connection now has a `close` method. You can call this when
 you are done with the connection to eagerly free resources. Currently
@@ -32,20 +33,20 @@ async def my_async_fn():
 It is not mandatory to call the `close` method. If you do not call it
 then the connection will be closed when the object is garbage collected.
 
-## Closeable Table
+### Closeable Table
 
 The Table now also has a `close` method, similar to the connection. This
 can be used to eagerly free the cache used by a Table object. Similar to
 the connection, it can be used as a context manager and it is not mandatory
 to call the `close` method.
 
-### Changes to Table APIs
+#### Changes to Table APIs
 
 - Previously `Table.schema` was a property. Now it is an async method.
 - The method `Table.__len__` was removed and `len(table)` will no longer
   work. Use `Table.count_rows` instead.
 
-### Creating Indices
+#### Creating Indices
 
 The `Table.create_index` method is now used for creating both vector indices
 and scalar indices. It currently requires a column name to be specified (the
@@ -55,12 +56,12 @@ the size of the data.
 To specify index configuration details you will need to specify which kind of
 index you are using.
 
-### Querying
+#### Querying
 
 The `Table.search` method has been renamed to `AsyncTable.vector_search` for
 clarity.
 
-## Features not yet supported
+### Features not yet supported
 
 The following features are not yet supported by the asynchronous API. However,
 we plan to support them soon.
@@ -74,3 +75,117 @@ we plan to support them soon.
   search
 - Remote connections to LanceDb Cloud are not yet supported.
 - The method Table.head is not yet supported.
+
+## TypeScript/JavaScript
+
+For JS/TS users, we offer a brand new SDK [@lancedb/lancedb](https://www.npmjs.com/package/@lancedb/lancedb)
+
+We tried to keep the API as similar as possible to the previous version, but there are a few small changes. Here are the most important ones:
+
+### Creating Tables
+
+[CreateTableOptions.writeOptions.writeMode](./javascript/interfaces/WriteOptions.md#writemode) has been replaced with [CreateTableOptions.mode](./js/interfaces/CreateTableOptions.md#mode)
+
+=== "vectordb (deprecated)"
+
+    ```ts
+    db.createTable(tableName, data, { writeMode: lancedb.WriteMode.Overwrite });
+    ```
+
+=== "@lancedb/lancedb"
+
+    ```ts
+    db.createTable(tableName, data, { mode: "overwrite" })
+    ```
+
+### Changes to Table APIs
+
+Previously `Table.schema` was a property. Now it is an async method.
+
+#### Creating Indices
+
+The `Table.createIndex` method is now used for creating both vector indices
+and scalar indices. It currently requires a column name to be specified (the
+column to index). Vector index defaults are now smarter and scale better with
+the size of the data.
+
+=== "vectordb (deprecated)"
+
+    ```ts
+    await tbl.createIndex({
+      column: "vector", // default
+      type: "ivf_pq",
+      num_partitions: 2,
+      num_sub_vectors: 2,
+    });
+    ```
+
+=== "@lancedb/lancedb"
+
+    ```ts
+    await table.createIndex("vector", {
+      config: lancedb.Index.ivfPq({
+        numPartitions: 2,
+        numSubVectors: 2,
+      }),
+    });
+    ```
+
+### Embedding Functions
+
+The embedding API has been completely reworked, and it now more closely resembles the Python API, including the new [embedding registry](./js/classes/embedding.EmbeddingFunctionRegistry.md)
+
+=== "vectordb (deprecated)"
+
+    ```ts
+
+    const embeddingFunction = new lancedb.OpenAIEmbeddingFunction('text', API_KEY)
+    const data = [
+        { id: 1, text: 'Black T-Shirt', price: 10 },
+        { id: 2, text: 'Leather Jacket', price: 50 }
+    ]
+    const table = await db.createTable('vectors', data, embeddingFunction)
+    ```
+
+=== "@lancedb/lancedb"
+
+    ```ts
+    import * as lancedb from "@lancedb/lancedb";
+    import * as arrow from "apache-arrow";
+    import { LanceSchema, getRegistry } from "@lancedb/lancedb/embedding";
+
+    const func = getRegistry().get("openai").create({apiKey: API_KEY});
+
+    const data = [
+        { id: 1, text: 'Black T-Shirt', price: 10 },
+        { id: 2, text: 'Leather Jacket', price: 50 }
+    ]
+
+    const table = await db.createTable('vectors', data, {
+        embeddingFunction: {
+            sourceColumn: "text",
+            function: func,
+        }
+    })
+
+    ```
+
+You can also use a schema driven approach, which parallels the Pydantic integration in our Python SDK:
+
+```ts
+const func = getRegistry().get("openai").create({apiKey: API_KEY});
+
+const data = [
+    { id: 1, text: 'Black T-Shirt', price: 10 },
+    { id: 2, text: 'Leather Jacket', price: 50 }
+]
+const schema = LanceSchema({
+    id: new arrow.Int32(),
+    text: func.sourceField(new arrow.Utf8()),
+    price: new arrow.Float64(),
+    vector: func.vectorField()
+})
+
+const table = await db.createTable('vectors', data, {schema})
+
+```

docs/src/notebooks/LlamaIndex_example.ipynb

@@ -0,0 +1,538 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "2db56c9b",
+   "metadata": {},
+   "source": [
+    "<a href=\"https://colab.research.google.com/github/run-llama/llama_index/blob/main/docs/docs/examples/vector_stores/LanceDBIndexDemo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "db0855d0",
+   "metadata": {},
+   "source": [
+    "# LanceDB Vector Store\n",
+    "In this notebook we are going to show how to use [LanceDB](https://www.lancedb.com) to perform vector searches in LlamaIndex"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "f44170b2",
+   "metadata": {},
+   "source": [
+    "If you're opening this Notebook on colab, you will probably need to install LlamaIndex 🦙."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6c84199c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install llama-index llama-index-vector-stores-lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1a90ce34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install lancedb==0.6.13 #Only required if the above cell installs an older version of lancedb (pypi package may not be released yet)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39c62671",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Refresh vector store URI if restarting or re-using the same notebook\n",
+    "! rm -rf ./lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "59b54276",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import logging\n",
+    "import sys\n",
+    "\n",
+    "# Uncomment to see debug logs\n",
+    "# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)\n",
+    "# logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))\n",
+    "\n",
+    "\n",
+    "from llama_index.core import SimpleDirectoryReader, Document, StorageContext\n",
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.vector_stores.lancedb import LanceDBVectorStore\n",
+    "import textwrap"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "26c71b6d",
+   "metadata": {},
+   "source": [
+    "### Setup OpenAI\n",
+    "The first step is to configure the openai key. It will be used to created embeddings for the documents loaded into the index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67b86621",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import openai\n",
+    "\n",
+    "openai.api_key = \"sk-\""
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "073f0a68",
+   "metadata": {},
+   "source": [
+    "Download Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eef1b911",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "--2024-06-11 16:42:37--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt\n",
+      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.110.133, 185.199.108.133, ...\n",
+      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
+      "HTTP request sent, awaiting response... 200 OK\n",
+      "Length: 75042 (73K) [text/plain]\n",
+      "Saving to: ‘data/paul_graham/paul_graham_essay.txt’\n",
+      "\n",
+      "data/paul_graham/pa 100%[===================>]  73.28K  --.-KB/s    in 0.02s   \n",
+      "\n",
+      "2024-06-11 16:42:37 (3.97 MB/s) - ‘data/paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "!mkdir -p 'data/paul_graham/'\n",
+    "!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt'"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "f7010b1d-d1bb-4f08-9309-a328bb4ea396",
+   "metadata": {},
+   "source": [
+    "### Loading documents\n",
+    "Load the documents stored in the `data/paul_graham/` using the SimpleDirectoryReader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c154dd4b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Document ID: cac1ba78-5007-4cf8-89ba-280264790115 Document Hash: fe2d4d3ef3a860780f6c2599808caa587c8be6516fe0ba4ca53cf117044ba953\n"
+     ]
+    }
+   ],
+   "source": [
+    "documents = SimpleDirectoryReader(\"./data/paul_graham/\").load_data()\n",
+    "print(\"Document ID:\", documents[0].doc_id, \"Document Hash:\", documents[0].hash)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "c0232fd1",
+   "metadata": {},
+   "source": [
+    "### Create the index\n",
+    "Here we create an index backed by LanceDB using the documents loaded previously. LanceDBVectorStore takes a few arguments.\n",
+    "- uri (str, required): Location where LanceDB will store its files.\n",
+    "- table_name (str, optional): The table name where the embeddings will be stored. Defaults to \"vectors\".\n",
+    "- nprobes (int, optional): The number of probes used. A higher number makes search more accurate but also slower. Defaults to 20.\n",
+    "- refine_factor: (int, optional): Refine the results by reading extra elements and re-ranking them in memory. Defaults to None\n",
+    "\n",
+    "- More details can be found at [LanceDB docs](https://lancedb.github.io/lancedb/ann_indexes)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1f2e20ef",
+   "metadata": {},
+   "source": [
+    "##### For LanceDB cloud :\n",
+    "```python\n",
+    "vector_store = LanceDBVectorStore( \n",
+    "    uri=\"db://db_name\", # your remote DB URI\n",
+    "    api_key=\"sk_..\", # lancedb cloud api key\n",
+    "    region=\"your-region\" # the region you configured\n",
+    "    ...\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8731da62",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vector_store = LanceDBVectorStore(\n",
+    "    uri=\"./lancedb\", mode=\"overwrite\", query_type=\"hybrid\"\n",
+    ")\n",
+    "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "\n",
+    "index = VectorStoreIndex.from_documents(\n",
+    "    documents, storage_context=storage_context\n",
+    ")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "8ee4473a-094f-4d0a-a825-e1213db07240",
+   "metadata": {},
+   "source": [
+    "### Query the index\n",
+    "We can now ask questions using our index. We can use filtering via `MetadataFilters` or use native lance `where` clause."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5eb6419b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.vector_stores import (\n",
+    "    MetadataFilters,\n",
+    "    FilterOperator,\n",
+    "    FilterCondition,\n",
+    "    MetadataFilter,\n",
+    ")\n",
+    "\n",
+    "from datetime import datetime\n",
+    "\n",
+    "\n",
+    "query_filters = MetadataFilters(\n",
+    "    filters=[\n",
+    "        MetadataFilter(\n",
+    "            key=\"creation_date\",\n",
+    "            operator=FilterOperator.EQ,\n",
+    "            value=datetime.now().strftime(\"%Y-%m-%d\"),\n",
+    "        ),\n",
+    "        MetadataFilter(\n",
+    "            key=\"file_size\", value=75040, operator=FilterOperator.GT\n",
+    "        ),\n",
+    "    ],\n",
+    "    condition=FilterCondition.AND,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ee201930",
+   "metadata": {},
+   "source": [
+    "### Hybrid Search\n",
+    "\n",
+    "LanceDB offers hybrid search with reranking capabilities. For complete documentation, refer [here](https://lancedb.github.io/lancedb/hybrid_search/hybrid_search/).\n",
+    "\n",
+    "This example uses the `colbert` reranker. The following cell installs the necessary dependencies for `colbert`. If you choose a different reranker, make sure to adjust the dependencies accordingly."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e12d1454",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U torch transformers tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c742cb07",
+   "metadata": {},
+   "source": [
+    "if you want to add a reranker at vector store initialization, you can pass it in the arguments like below :\n",
+    "```\n",
+    "from lancedb.rerankers import ColbertReranker\n",
+    "reranker = ColbertReranker()\n",
+    "vector_store = LanceDBVectorStore(uri=\"./lancedb\", reranker=reranker, mode=\"overwrite\")\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27ea047b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8414517f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from lancedb.rerankers import ColbertReranker\n",
+    "\n",
+    "reranker = ColbertReranker()\n",
+    "vector_store._add_reranker(reranker)\n",
+    "\n",
+    "query_engine = index.as_query_engine(\n",
+    "    filters=query_filters,\n",
+    "    # vector_store_kwargs={\n",
+    "    #     \"query_type\": \"fts\",\n",
+    "    # },\n",
+    ")\n",
+    "\n",
+    "response = query_engine.query(\"How much did Viaweb charge per month?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc6ccb7a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Viaweb charged $100 a month for a small store and $300 a month for a big one.\n",
+      "metadata - {'65ed5f07-5b8a-4143-a939-e8764884828e': {'file_path': '/Users/raghavdixit/Desktop/open_source/llama_index_lance/docs/docs/examples/vector_stores/data/paul_graham/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-11', 'last_modified_date': '2024-06-11'}, 'be231827-20b8-4988-ac75-94fa79b3c22e': {'file_path': '/Users/raghavdixit/Desktop/open_source/llama_index_lance/docs/docs/examples/vector_stores/data/paul_graham/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-11', 'last_modified_date': '2024-06-11'}}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(response)\n",
+    "print(\"metadata -\", response.metadata)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0c1c6c73",
+   "metadata": {},
+   "source": [
+    "##### lance filters(SQL like) directly via the `where` clause :"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a2bcc07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lance_filter = \"metadata.file_name = 'paul_graham_essay.txt' \"\n",
+    "retriever = index.as_retriever(vector_store_kwargs={\"where\": lance_filter})\n",
+    "response = retriever.retrieve(\"What did the author do growing up?\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7ac47cf9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "What I Worked On\n",
+      "\n",
+      "February 2021\n",
+      "\n",
+      "Before college the two main things I worked on, outside of school, were writing and programming. I didn't write essays. I wrote what beginning writers were supposed to write then, and probably still are: short stories. My stories were awful. They had hardly any plot, just characters with strong feelings, which I imagined made them deep.\n",
+      "\n",
+      "The first programs I tried writing were on the IBM 1401 that our school district used for what was then called \"data processing.\" This was in 9th grade, so I was 13 or 14. The school district's 1401 happened to be in the basement of our junior high school, and my friend Rich Draves and I got permission to use it. It was like a mini Bond villain's lair down there, with all these alien-looking machines — CPU, disk drives, printer, card reader — sitting up on a raised floor under bright fluorescent lights.\n",
+      "\n",
+      "The language we used was an early version of Fortran. You had to type programs on punch cards, then stack them in the card reader and press a button to load the program into memory and run it. The result would ordinarily be to print something on the spectacularly loud printer.\n",
+      "\n",
+      "I was puzzled by the 1401. I couldn't figure out what to do with it. And in retrospect there's not much I could have done with it. The only form of input to programs was data stored on punched cards, and I didn't have any data stored on punched cards. The only other option was to do things that didn't rely on any input, like calculate approximations of pi, but I didn't know enough math to do anything interesting of that type. So I'm not surprised I can't remember any programs I wrote, because they can't have done much. My clearest memory is of the moment I learned it was possible for programs not to terminate, when one of mine didn't. On a machine without time-sharing, this was a social as well as a technical error, as the data center manager's expression made clear.\n",
+      "\n",
+      "With microcomputers, everything changed. Now you could have a computer sitting right in front of you, on a desk, that could respond to your keystrokes as it was running instead of just churning through a stack of punch cards and then stopping. [1]\n",
+      "\n",
+      "The first of my friends to get a microcomputer built it himself. It was sold as a kit by Heathkit. I remember vividly how impressed and envious I felt watching him sitting in front of it, typing programs right into the computer.\n",
+      "\n",
+      "Computers were expensive in those days and it took me years of nagging before I convinced my father to buy one, a TRS-80, in about 1980. The gold standard then was the Apple II, but a TRS-80 was good enough. This was when I really started programming. I wrote simple games, a program to predict how high my model rockets would fly, and a word processor that my father used to write at least one book. There was only room in memory for about 2 pages of text, so he'd write 2 pages at a time and then print them out, but it was a lot better than a typewriter.\n",
+      "\n",
+      "Though I liked programming, I didn't plan to study it in college. In college I was going to study philosophy, which sounded much more powerful. It seemed, to my naive high school self, to be the study of the ultimate truths, compared to which the things studied in other fields would be mere domain knowledge. What I discovered when I got to college was that the other fields took up so much of the space of ideas that there wasn't much left for these supposed ultimate truths. All that seemed left for philosophy were edge cases that people in other fields felt could safely be ignored.\n",
+      "\n",
+      "I couldn't have put this into words when I was 18. All I knew at the time was that I kept taking philosophy courses and they kept being boring. So I decided to switch to AI.\n",
+      "\n",
+      "AI was in the air in the mid 1980s, but there were two things especially that made me want to work on it: a novel by Heinlein called The Moon is a Harsh Mistress, which featured an intelligent computer called Mike, and a PBS documentary that showed Terry Winograd using SHRDLU. I haven't tried rereading The Moon is a Harsh Mistress, so I don't know how well it has aged, but when I read it I was drawn entirely into its world.\n",
+      "metadata - {'file_path': '/Users/raghavdixit/Desktop/open_source/llama_index_lance/docs/docs/examples/vector_stores/data/paul_graham/paul_graham_essay.txt', 'file_name': 'paul_graham_essay.txt', 'file_type': 'text/plain', 'file_size': 75042, 'creation_date': '2024-06-11', 'last_modified_date': '2024-06-11'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(response[0].get_content())\n",
+    "print(\"metadata -\", response[0].metadata)"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "id": "6afc84ac",
+   "metadata": {},
+   "source": [
+    "### Appending data\n",
+    "You can also add data to an existing index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "759a532e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nodes = [node.node for node in response]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "069fc099",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del index\n",
+    "\n",
+    "index = VectorStoreIndex.from_documents(\n",
+    "    [Document(text=\"The sky is purple in Portland, Maine\")],\n",
+    "    uri=\"/tmp/new_dataset\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a64ed441",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.insert_nodes(nodes)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b5cffcfe",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Portland, Maine\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "response = query_engine.query(\"Where is the sky purple?\")\n",
+    "print(textwrap.fill(str(response), 100))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ec548a02",
+   "metadata": {},
+   "source": [
+    "You can also create an index from an existing table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dc99404d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del index\n",
+    "\n",
+    "vec_store = LanceDBVectorStore.from_table(vector_store._table)\n",
+    "index = VectorStoreIndex.from_vector_store(vec_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7b2e8cca",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The author started Viaweb and Aspra.\n"
+     ]
+    }
+   ],
+   "source": [
+    "query_engine = index.as_query_engine()\n",
+    "response = query_engine.query(\"What companies did the author start?\")\n",
+    "print(textwrap.fill(str(response), 100))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/src/notebooks/langchain_example.ipynb

@@ -0,0 +1,566 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "683953b3",
+   "metadata": {},
+   "source": [
+    "# LanceDB\n",
+    "\n",
+    ">[LanceDB](https://lancedb.com/) is an open-source database for vector-search built with persistent storage, which greatly simplifies retrevial, filtering and management of embeddings. Fully open source.\n",
+    "\n",
+    "This notebook shows how to use functionality related to the `LanceDB` vector database based on the Lance data format."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b1051ba9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install tantivy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "88ac92c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U langchain-openai langchain-community"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a1c84d6-a10f-428c-95cd-46d3a1702e07",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install lancedb"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "99134dd1-b91e-486f-8d90-534248e43b9d",
+   "metadata": {},
+   "source": [
+    "We want to use OpenAIEmbeddings so we have to get the OpenAI API Key. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "a0361f5c-e6f4-45f4-b829-11680cf03cec",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "d114ed78",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! rm -rf /tmp/lancedb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "a3c3999a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.document_loaders import TextLoader\n",
+    "from langchain_community.vectorstores import LanceDB\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
+    "from langchain_text_splitters import CharacterTextSplitter\n",
+    "\n",
+    "loader = TextLoader(\"../../how_to/state_of_the_union.txt\")\n",
+    "documents = loader.load()\n",
+    "\n",
+    "documents = CharacterTextSplitter().split_documents(documents)\n",
+    "embeddings = OpenAIEmbeddings()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e9517bb0",
+   "metadata": {},
+   "source": [
+    "##### For LanceDB cloud, you can invoke the vector store as follows :\n",
+    "\n",
+    "\n",
+    "```python\n",
+    "db_url = \"db://lang_test\" # url of db you created\n",
+    "api_key = \"xxxxx\" # your API key\n",
+    "region=\"us-east-1-dev\"  # your selected region\n",
+    "\n",
+    "vector_store = LanceDB(\n",
+    "    uri=db_url,\n",
+    "    api_key=api_key,\n",
+    "    region=region,\n",
+    "    embedding=embeddings,\n",
+    "    table_name='langchain_test'\n",
+    "    )\n",
+    "```\n",
+    "\n",
+    "You can also add `region`, `api_key`, `uri` to `from_documents()` classmethod\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "6e104aee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from lancedb.rerankers import LinearCombinationReranker\n",
+    "\n",
+    "reranker = LinearCombinationReranker(weight=0.3)\n",
+    "\n",
+    "docsearch = LanceDB.from_documents(documents, embeddings, reranker=reranker)\n",
+    "query = \"What did the president say about Ketanji Brown Jackson\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "259c7988",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "relevance score -  0.7066475030191711\n",
+      "text-  They were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \n",
+      "\n",
+      "Officer Mora was 27 years old. \n",
+      "\n",
+      "Officer Rivera was 22. \n",
+      "\n",
+      "Both Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \n",
+      "\n",
+      "I spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves. \n",
+      "\n",
+      "I’ve worked on these issues a long time. \n",
+      "\n",
+      "I know what works: Investing in crime prevention and community police officers who’ll walk the beat, who’ll know the neighborhood, and who can restore trust and safety. \n",
+      "\n",
+      "So let’s not abandon our streets. Or choose between safety and equal justice. \n",
+      "\n",
+      "Let’s come together to protect our communities, restore trust, and hold law enforcement accountable. \n",
+      "\n",
+      "That’s why the Justice Department required body cameras, banned chokeholds, and restricted no-knock warrants for its officers. \n",
+      "\n",
+      "That’s why the American Rescue \n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = docsearch.similarity_search_with_relevance_scores(query)\n",
+    "print(\"relevance score - \", docs[0][1])\n",
+    "print(\"text- \", docs[0][0].page_content[:1000])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "9fa29dae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "distance -  0.30000001192092896\n",
+      "text-  My administration is providing assistance with job training and housing, and now helping lower-income veterans get VA care debt-free.  \n",
+      "\n",
+      "Our troops in Iraq and Afghanistan faced many dangers. \n",
+      "\n",
+      "One was stationed at bases and breathing in toxic smoke from “burn pits” that incinerated wastes of war—medical and hazard material, jet fuel, and more. \n",
+      "\n",
+      "When they came home, many of the world’s fittest and best trained warriors were never the same. \n",
+      "\n",
+      "Headaches. Numbness. Dizziness. \n",
+      "\n",
+      "A cancer that would put them in a flag-draped coffin. \n",
+      "\n",
+      "I know. \n",
+      "\n",
+      "One of those soldiers was my son Major Beau Biden. \n",
+      "\n",
+      "We don’t know for sure if a burn pit was the cause of his brain cancer, or the diseases of so many of our troops. \n",
+      "\n",
+      "But I’m committed to finding out everything we can. \n",
+      "\n",
+      "Committed to military families like Danielle Robinson from Ohio. \n",
+      "\n",
+      "The widow of Sergeant First Class Heath Robinson.  \n",
+      "\n",
+      "He was born a soldier. Army National Guard. Combat medic in Kosovo and Iraq. \n",
+      "\n",
+      "Stationed near Baghdad, just ya\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = docsearch.similarity_search_with_score(query=\"Headaches\", query_type=\"hybrid\")\n",
+    "print(\"distance - \", docs[0][1])\n",
+    "print(\"text- \", docs[0][0].page_content[:1000])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "e70ad201",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "reranker :  <lancedb.rerankers.linear_combination.LinearCombinationReranker object at 0x107ef1130>\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"reranker : \", docsearch._reranker)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f5e1cdfd",
+   "metadata": {},
+   "source": [
+    "Additionaly, to explore the table you can load it into a df or save it in a csv file: \n",
+    "```python\n",
+    "tbl = docsearch.get_table()\n",
+    "print(\"tbl:\", tbl)\n",
+    "pd_df = tbl.to_pandas()\n",
+    "# pd_df.to_csv(\"docsearch.csv\", index=False)\n",
+    "\n",
+    "# you can also create a new vector store object using an older connection object:\n",
+    "vector_store = LanceDB(connection=tbl, embedding=embeddings)\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "9c608226",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "metadata : {'source': '../../how_to/state_of_the_union.txt'}\n",
+      "\n",
+      "SQL filtering :\n",
+      "\n",
+      "They were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \n",
+      "\n",
+      "Officer Mora was 27 years old. \n",
+      "\n",
+      "Officer Rivera was 22. \n",
+      "\n",
+      "Both Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \n",
+      "\n",
+      "I spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves. \n",
+      "\n",
+      "I’ve worked on these issues a long time. \n",
+      "\n",
+      "I know what works: Investing in crime prevention and community police officers who’ll walk the beat, who’ll know the neighborhood, and who can restore trust and safety. \n",
+      "\n",
+      "So let’s not abandon our streets. Or choose between safety and equal justice. \n",
+      "\n",
+      "Let’s come together to protect our communities, restore trust, and hold law enforcement accountable. \n",
+      "\n",
+      "That’s why the Justice Department required body cameras, banned chokeholds, and restricted no-knock warrants for its officers. \n",
+      "\n",
+      "That’s why the American Rescue Plan provided $350 Billion that cities, states, and counties can use to hire more police and invest in proven strategies like community violence interruption—trusted messengers breaking the cycle of violence and trauma and giving young people hope.  \n",
+      "\n",
+      "We should all agree: The answer is not to Defund the police. The answer is to FUND the police with the resources and training they need to protect our communities. \n",
+      "\n",
+      "I ask Democrats and Republicans alike: Pass my budget and keep our neighborhoods safe.  \n",
+      "\n",
+      "And I will keep doing everything in my power to crack down on gun trafficking and ghost guns you can buy online and make at home—they have no serial numbers and can’t be traced. \n",
+      "\n",
+      "And I ask Congress to pass proven measures to reduce gun violence. Pass universal background checks. Why should anyone on a terrorist list be able to purchase a weapon? \n",
+      "\n",
+      "Ban assault weapons and high-capacity magazines. \n",
+      "\n",
+      "Repeal the liability shield that makes gun manufacturers the only industry in America that can’t be sued. \n",
+      "\n",
+      "These laws don’t infringe on the Second Amendment. They save lives. \n",
+      "\n",
+      "The most fundamental right in America is the right to vote – and to have it counted. And it’s under assault. \n",
+      "\n",
+      "In state after state, new laws have been passed, not only to suppress the vote, but to subvert entire elections. \n",
+      "\n",
+      "We cannot let this happen. \n",
+      "\n",
+      "Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. \n",
+      "\n",
+      "Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. \n",
+      "\n",
+      "One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. \n",
+      "\n",
+      "And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. \n",
+      "\n",
+      "A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. \n",
+      "\n",
+      "And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. \n",
+      "\n",
+      "We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling.  \n",
+      "\n",
+      "We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers.  \n",
+      "\n",
+      "We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster.\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = docsearch.similarity_search(\n",
+    "    query=query, filter={\"metadata.source\": \"../../how_to/state_of_the_union.txt\"}\n",
+    ")\n",
+    "\n",
+    "print(\"metadata :\", docs[0].metadata)\n",
+    "\n",
+    "# or you can directly supply SQL string filters :\n",
+    "\n",
+    "print(\"\\nSQL filtering :\\n\")\n",
+    "docs = docsearch.similarity_search(query=query, filter=\"text LIKE '%Officer Rivera%'\")\n",
+    "print(docs[0].page_content)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9a173c94",
+   "metadata": {},
+   "source": [
+    "## Adding images "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05f669d7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U langchain-experimental"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ed69810",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install open_clip_torch torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "2cacb5ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! rm -rf '/tmp/multimmodal_lance'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "b3456e2c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_experimental.open_clip import OpenCLIPEmbeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "3848eba2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "import requests\n",
+    "\n",
+    "# List of image URLs to download\n",
+    "image_urls = [\n",
+    "    \"https://github.com/raghavdixit99/assets/assets/34462078/abf47cc4-d979-4aaa-83be-53a2115bf318\",\n",
+    "    \"https://github.com/raghavdixit99/assets/assets/34462078/93be928e-522b-4e37-889d-d4efd54b2112\",\n",
+    "]\n",
+    "\n",
+    "texts = [\"bird\", \"dragon\"]\n",
+    "\n",
+    "# Directory to save images\n",
+    "dir_name = \"./photos/\"\n",
+    "\n",
+    "# Create directory if it doesn't exist\n",
+    "os.makedirs(dir_name, exist_ok=True)\n",
+    "\n",
+    "image_uris = []\n",
+    "# Download and save each image\n",
+    "for i, url in enumerate(image_urls, start=1):\n",
+    "    response = requests.get(url)\n",
+    "    path = os.path.join(dir_name, f\"image{i}.jpg\")\n",
+    "    image_uris.append(path)\n",
+    "    with open(path, \"wb\") as f:\n",
+    "        f.write(response.content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "3d62c2a0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_community.vectorstores import LanceDB\n",
+    "\n",
+    "vec_store = LanceDB(\n",
+    "    table_name=\"multimodal_test\",\n",
+    "    embedding=OpenCLIPEmbeddings(),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "id": "ebbb4881",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['b673620b-01f0-42ca-a92e-d033bb92c0a6',\n",
+       " '99c3a5b0-b577-417a-8177-92f4a655dbfb']"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store.add_images(uris=image_uris)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "3c29dea3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['f7adde5d-a4a3-402b-9e73-088b230722c3',\n",
+       " 'cbed59da-0aec-4bff-8820-9e59d81a2140']"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store.add_texts(texts)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "8b2f25ce",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "img_embed = vec_store._embedding.embed_query(\"bird\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "87a24079",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Document(page_content='bird', metadata={'id': 'f7adde5d-a4a3-402b-9e73-088b230722c3'})"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store.similarity_search_by_vector(img_embed)[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "78557867",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LanceTable(connection=LanceDBConnection(/tmp/lancedb), name=\"multimodal_test\")"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vec_store._table"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/src/python/saas-python.md

@@ -1,6 +1,6 @@
 # Python API Reference (SaaS)
 
-This section contains the API reference for the SaaS Python API.
+This section contains the API reference for the LanceDB Cloud Python API.
 
 ## Installation
 

docs/src/reranking/index.md

@@ -15,7 +15,6 @@ LanceDB comes with some built-in rerankers. Some of the rerankers that are avail
 Using rerankers is optional for vector and FTS. However, for hybrid search, rerankers are required. To use a reranker, you need to create an instance of the reranker and pass it to the `rerank` method of the query builder.
 
 ```python
-import numpy
 import lancedb
 from lancedb.embeddings import get_registry
 from lancedb.pydantic import LanceModel, Vector
@@ -54,6 +53,7 @@ LanceDB comes with some built-in rerankers. Here are some of the rerankers that
 - [ColBERT Reranker](./colbert.md)
 - [OpenAI Reranker](./openai.md)
 - [Linear Combination Reranker](./linear_combination.md)
+- [Jina Reranker](./jina.md)
 
 ## Creating Custom Rerankers
 

docs/src/reranking/jina.md

@@ -0,0 +1,78 @@
+# Jina Reranker
+
+This re-ranker uses the [Jina](https://jina.ai/reranker/) API to rerank the search results. You can use this re-ranker by passing `JinaReranker()` to the `rerank()` method. Note that you'll either need to set the `JINA_API_KEY` environment variable or pass the `api_key` argument to use this re-ranker.
+
+
+!!! note
+    Supported Query Types: Hybrid, Vector, FTS
+
+
+```python
+import os
+import lancedb
+from lancedb.embeddings import get_registry
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.rerankers import JinaReranker
+
+os.environ['JINA_API_KEY'] = "jina_*"
+
+
+embedder = get_registry().get("jina").create()
+db = lancedb.connect("~/.lancedb")
+
+class Schema(LanceModel):
+    text: str = embedder.SourceField()
+    vector: Vector(embedder.ndims()) = embedder.VectorField()
+
+data = [
+    {"text": "hello world"},
+    {"text": "goodbye world"}
+    ]
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+tbl.add(data)
+reranker = JinaReranker(api_key="key")
+
+# Run vector search with a reranker
+result = tbl.search("hello").rerank(reranker=reranker).to_list() 
+
+# Run FTS search with a reranker
+result = tbl.search("hello", query_type="fts").rerank(reranker=reranker).to_list()
+
+# Run hybrid search with a reranker
+tbl.create_fts_index("text", replace=True)
+result = tbl.search("hello", query_type="hybrid").rerank(reranker=reranker).to_list()
+
+```
+
+Accepted Arguments
+----------------
+| Argument | Type | Default | Description |
+| --- | --- | --- | --- |
+| `model_name` | `str` | `"jina-reranker-v2-base-multilingual"` | The name of the reranker model to use. You can find the list of available models in https://jina.ai/reranker/|
+| `column` | `str` | `"text"` | The name of the column to use as input to the cross encoder model. |
+| `top_n` | `str` | `None` | The number of results to return. If None, will return all results. |
+| `api_key` | `str` | `None` | The API key for the Jina API. If not provided, the `JINA_API_KEY` environment variable is used. |
+| `return_score` | str | `"relevance"` | Options are "relevance" or "all". The type of score to return. If "relevance", will return only the `_relevance_score. If "all" is supported, will return relevance score along with the vector and/or fts scores depending on query type |
+
+
+
+## Supported Scores for each query type
+You can specify the type of scores you want the reranker to return. The following are the supported scores for each query type:
+
+### Hybrid Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ❌ Not Supported | Returns have vector(`_distance`) and FTS(`score`) along with Hybrid Search score(`_relevance_score`) |
+
+### Vector Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have vector(`_distance`) along with Hybrid Search score(`_relevance_score`) |
+
+### FTS Search
+|`return_score`| Status | Description |
+| --- | --- | --- |
+| `relevance` | ✅ Supported | Returns only have the `_relevance_score` column |
+| `all` | ✅ Supported | Returns have FTS(`score`) along with Hybrid Search score(`_relevance_score`) |

docs/src/search.md

@@ -53,13 +53,24 @@ db.create_table("my_vectors", data=data)
         .to_list()
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/search_legacy.ts:import"
 
-    --8<-- "docs/src/search_legacy.ts:search1"
+        ```ts
-    ```
+        --8<-- "nodejs/examples/search.ts:import"
+
+        --8<-- "nodejs/examples/search.ts:search1"
+        ```
+
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/search_legacy.ts:import"
+
+        --8<-- "docs/src/search_legacy.ts:search1"
+        ```
 
 By default, `l2` will be used as metric type. You can specify the metric type as
 `cosine` or `dot` if required.
@@ -73,11 +84,19 @@ By default, `l2` will be used as metric type. You can specify the metric type as
         .to_list()
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/search_legacy.ts:search2"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/search.ts:search2"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```javascript
+        --8<-- "docs/src/search_legacy.ts:search2"
+        ```
 
 ## Approximate nearest neighbor (ANN) search
 

docs/src/sql.md

@@ -44,11 +44,19 @@ const tbl = await db.createTable('myVectors', data)
     )
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/sql_legacy.ts:search"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/filtering.ts:search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/sql_legacy.ts:search"
+        ```
 
 ## SQL filters
 
@@ -78,11 +86,19 @@ For example, the following filter string is acceptable:
        .to_arrow()
     ```
 
-=== "Javascript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<-- "docs/src/sql_legacy.ts:vec_search"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/filtering.ts:vec_search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<-- "docs/src/sql_legacy.ts:vec_search"
+        ```
 
 If your column name contains special characters or is a [SQL Keyword](https://docs.rs/sqlparser/latest/sqlparser/keywords/index.html),
 you can use backtick (`` ` ``) to escape it. For nested fields, each segment of the
@@ -148,10 +164,18 @@ You can also filter your data without search.
     tbl.search().where("id = 10").limit(10).to_arrow()
     ```
 
-=== "JavaScript"
+=== "TypeScript"
 
-    ```javascript
+    === "@lancedb/lancedb"
-    --8<---- "docs/src/sql_legacy.ts:sql_search"
+
-    ```
+        ```ts
+        --8<-- "nodejs/examples/filtering.ts:sql_search"
+        ```
+
+    === "vectordb (deprecated)"
+
+        ```ts
+        --8<---- "docs/src/sql_legacy.ts:sql_search"
+        ```
 
 !!!warning "If your table is large, this could potentially return a very large amount of data. Please be sure to use a `limit` clause unless you're sure you want to return the whole result set."

node/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.5.2",
+  "version": "0.7.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "vectordb",
-      "version": "0.5.2",
+      "version": "0.7.1",
       "cpu": [
         "x64",
         "arm64"

node/package.json

@@ -1,12 +1,12 @@
 {
   "name": "vectordb",
-  "version": "0.5.2",
+  "version": "0.7.1",
   "description": " Serverless, low-latency vector database for AI applications",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "scripts": {
     "tsc": "tsc -b",
-    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb_node index.node -- cargo build --message-format=json",
+    "build": "npm run tsc && cargo-cp-artifact --artifact cdylib lancedb_node index.node -- cargo build -p lancedb-node --message-format=json",
     "build-release": "npm run build -- --release",
     "test": "npm run tsc && mocha -recursive dist/test",
     "integration-test": "npm run tsc && mocha -recursive dist/integration_test",

node/src/index.ts

@@ -62,6 +62,8 @@ export {
 
 const defaultAwsRegion = "us-west-2";
 
+const defaultRequestTimeout = 10_000
+
 export interface AwsCredentials {
   accessKeyId: string
 
@@ -119,6 +121,11 @@ export interface ConnectionOptions {
    */
   hostOverride?: string
 
+  /**
+   * Duration in milliseconds for request timeout. Default = 10,000 (10 seconds)
+   */
+  timeout?: number
+
   /**
    * (For LanceDB OSS only): The interval, in seconds, at which to check for
    * updates to the table from other processes. If None, then consistency is not
@@ -204,7 +211,8 @@ export async function connect(
           awsCredentials: undefined,
           awsRegion: defaultAwsRegion,
           apiKey: undefined,
-          region: defaultAwsRegion
+          region: defaultAwsRegion,
+          timeout: defaultRequestTimeout 
         },
         arg
       );

node/src/remote/client.ts

@@ -41,7 +41,7 @@ async function callWithMiddlewares (
     if (i > middlewares.length) {
       const headers = Object.fromEntries(req.headers.entries())
       const params = Object.fromEntries(req.params?.entries() ?? [])
-      const timeout = 10000
+      const timeout = opts?.timeout
       let res
       if (req.method === Method.POST) {
         res = await axios.post(
@@ -82,6 +82,7 @@ async function callWithMiddlewares (
 
 interface MiddlewareInvocationOptions {
   responseType?: ResponseType
+  timeout?: number,
 }
 
 /**
@@ -123,15 +124,19 @@ export class HttpLancedbClient {
   private readonly _url: string
   private readonly _apiKey: () => string
   private readonly _middlewares: HttpLancedbClientMiddleware[]
+  private readonly _timeout: number | undefined
 
   public constructor (
     url: string,
     apiKey: string,
-    private readonly _dbName?: string
+    timeout?: number,
+    private readonly _dbName?: string,
+    
   ) {
     this._url = url
     this._apiKey = () => apiKey
     this._middlewares = []
+    this._timeout = timeout
   }
 
   get uri (): string {
@@ -230,7 +235,10 @@ export class HttpLancedbClient {
 
     let response
     try {
-      response = await callWithMiddlewares(req, this._middlewares, { responseType })
+      response = await callWithMiddlewares(req, this._middlewares, {
+        responseType,
+        timeout: this._timeout,
+      })
 
       // return response
     } catch (err: any) {
@@ -267,7 +275,7 @@ export class HttpLancedbClient {
    * Make a clone of this client
    */
   private clone (): HttpLancedbClient {
-    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._dbName)
+    const clone = new HttpLancedbClient(this._url, this._apiKey(), this._timeout, this._dbName)
     for (const mw of this._middlewares) {
       clone._middlewares.push(mw)
     }

node/src/remote/index.ts

@@ -72,6 +72,7 @@ export class RemoteConnection implements Connection {
     this._client = new HttpLancedbClient(
       server,
       opts.apiKey,
+      opts.timeout,
       opts.hostOverride === undefined ? undefined : this._dbName
     )
   }

nodejs/Cargo.toml

@@ -15,11 +15,11 @@ crate-type = ["cdylib"]
 arrow-ipc.workspace = true
 futures.workspace = true
 lancedb = { path = "../rust/lancedb" }
-napi = { version = "2.15", default-features = false, features = [
+napi = { version = "2.16.8", default-features = false, features = [
-    "napi7",
+    "napi9",
     "async",
 ] }
-napi-derive = "2"
+napi-derive = "2.16.4"
 
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "*", features = ["static"] }

nodejs/__test__/arrow.test.ts

@@ -1,3 +1,4 @@
+import { Schema } from "apache-arrow";
 // Copyright 2024 Lance Developers.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,40 +13,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import {
+import * as arrow13 from "apache-arrow-13";
-  Binary,
+import * as arrow14 from "apache-arrow-14";
-  Bool,
+import * as arrow15 from "apache-arrow-15";
-  DataType,
+import * as arrow16 from "apache-arrow-16";
-  Dictionary,
+import * as arrow17 from "apache-arrow-17";
-  Field,
+
-  FixedSizeList,
-  Float,
-  Float16,
-  Float32,
-  Float64,
-  Int32,
-  Int64,
-  List,
-  MetadataVersion,
-  Precision,
-  Schema,
-  Struct,
-  type Table,
-  Type,
-  Utf8,
-  tableFromIPC,
-} from "apache-arrow";
-import {
-  Dictionary as OldDictionary,
-  Field as OldField,
-  FixedSizeList as OldFixedSizeList,
-  Float32 as OldFloat32,
-  Int32 as OldInt32,
-  Schema as OldSchema,
-  Struct as OldStruct,
-  TimestampNanosecond as OldTimestampNanosecond,
-  Utf8 as OldUtf8,
-} from "apache-arrow-old";
 import {
   convertToTable,
   fromTableToBuffer,
@@ -72,429 +45,520 @@ function sampleRecords(): Array<Record<string, any>> {
     },
   ];
 }
+describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
+  "Arrow",
+  (
+    arrow:
+      | typeof arrow13
+      | typeof arrow14
+      | typeof arrow15
+      | typeof arrow16
+      | typeof arrow17,
+  ) => {
+    type ApacheArrow =
+      | typeof arrow13
+      | typeof arrow14
+      | typeof arrow15
+      | typeof arrow16
+      | typeof arrow17;
+    const {
+      Schema,
+      Field,
+      Binary,
+      Bool,
+      Utf8,
+      Float64,
+      Struct,
+      List,
+      Int32,
+      Int64,
+      Float,
+      Float16,
+      Float32,
+      FixedSizeList,
+      Precision,
+      tableFromIPC,
+      DataType,
+      Dictionary,
+      // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    } = <any>arrow;
+    type Schema = ApacheArrow["Schema"];
+    type Table = ApacheArrow["Table"];
 
-// Helper method to verify various ways to create a table
+    // Helper method to verify various ways to create a table
-async function checkTableCreation(
+    async function checkTableCreation(
-  tableCreationMethod: (
+      tableCreationMethod: (
-    records: Record<string, unknown>[],
+        records: Record<string, unknown>[],
-    recordsReversed: Record<string, unknown>[],
+        recordsReversed: Record<string, unknown>[],
-    schema: Schema,
+        schema: Schema,
-  ) => Promise<Table>,
+      ) => Promise<Table>,
-  infersTypes: boolean,
+      infersTypes: boolean,
-): Promise<void> {
+    ): Promise<void> {
-  const records = sampleRecords();
+      const records = sampleRecords();
-  const recordsReversed = [
+      const recordsReversed = [
-    {
+        {
-      list: ["anime", "action", "comedy"],
+          list: ["anime", "action", "comedy"],
-      struct: { x: 0, y: 0 },
+          struct: { x: 0, y: 0 },
-      string: "hello",
+          string: "hello",
-      number: 7,
+          number: 7,
-      boolean: false,
+          boolean: false,
-      binary: Buffer.alloc(5),
+          binary: Buffer.alloc(5),
-    },
-  ];
-  const schema = new Schema([
-    new Field("binary", new Binary(), false),
-    new Field("boolean", new Bool(), false),
-    new Field("number", new Float64(), false),
-    new Field("string", new Utf8(), false),
-    new Field(
-      "struct",
-      new Struct([
-        new Field("x", new Float64(), false),
-        new Field("y", new Float64(), false),
-      ]),
-    ),
-    new Field("list", new List(new Field("item", new Utf8(), false)), false),
-  ]);
-
-  const table = await tableCreationMethod(records, recordsReversed, schema);
-  schema.fields.forEach((field, idx) => {
-    const actualField = table.schema.fields[idx];
-    // Type inference always assumes nullable=true
-    if (infersTypes) {
-      expect(actualField.nullable).toBe(true);
-    } else {
-      expect(actualField.nullable).toBe(false);
-    }
-    expect(table.getChild(field.name)?.type.toString()).toEqual(
-      field.type.toString(),
-    );
-    expect(table.getChildAt(idx)?.type.toString()).toEqual(
-      field.type.toString(),
-    );
-  });
-}
-
-describe("The function makeArrowTable", function () {
-  it("will use data types from a provided schema instead of inference", async function () {
-    const schema = new Schema([
-      new Field("a", new Int32()),
-      new Field("b", new Float32()),
-      new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
-      new Field("d", new Int64()),
-    ]);
-    const table = makeArrowTable(
-      [
-        { a: 1, b: 2, c: [1, 2, 3], d: 9 },
-        { a: 4, b: 5, c: [4, 5, 6], d: 10 },
-        { a: 7, b: 8, c: [7, 8, 9], d: null },
-      ],
-      { schema },
-    );
-
-    const buf = await fromTableToBuffer(table);
-    expect(buf.byteLength).toBeGreaterThan(0);
-
-    const actual = tableFromIPC(buf);
-    expect(actual.numRows).toBe(3);
-    const actualSchema = actual.schema;
-    expect(actualSchema).toEqual(schema);
-  });
-
-  it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
-    const schema = new Schema([
-      new Field("a", new Float(Precision.DOUBLE), true),
-      new Field("b", new Float(Precision.DOUBLE), true),
-      new Field(
-        "vector",
-        new FixedSizeList(
-          3,
-          new Field("item", new Float(Precision.SINGLE), true),
-        ),
-        true,
-      ),
-    ]);
-    const table = makeArrowTable([
-      { a: 1, b: 2, vector: [1, 2, 3] },
-      { a: 4, b: 5, vector: [4, 5, 6] },
-      { a: 7, b: 8, vector: [7, 8, 9] },
-    ]);
-
-    const buf = await fromTableToBuffer(table);
-    expect(buf.byteLength).toBeGreaterThan(0);
-
-    const actual = tableFromIPC(buf);
-    expect(actual.numRows).toBe(3);
-    const actualSchema = actual.schema;
-    expect(actualSchema).toEqual(schema);
-  });
-
-  it("can support multiple vector columns", async function () {
-    const schema = new Schema([
-      new Field("a", new Float(Precision.DOUBLE), true),
-      new Field("b", new Float(Precision.DOUBLE), true),
-      new Field(
-        "vec1",
-        new FixedSizeList(3, new Field("item", new Float16(), true)),
-        true,
-      ),
-      new Field(
-        "vec2",
-        new FixedSizeList(3, new Field("item", new Float16(), true)),
-        true,
-      ),
-    ]);
-    const table = makeArrowTable(
-      [
-        { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-        { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-        { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] },
-      ],
-      {
-        vectorColumns: {
-          vec1: { type: new Float16() },
-          vec2: { type: new Float16() },
         },
-      },
+      ];
-    );
+      const schema = new Schema([
-
+        new Field("binary", new Binary(), false),
-    const buf = await fromTableToBuffer(table);
+        new Field("boolean", new Bool(), false),
-    expect(buf.byteLength).toBeGreaterThan(0);
+        new Field("number", new Float64(), false),
-
+        new Field("string", new Utf8(), false),
-    const actual = tableFromIPC(buf);
+        new Field(
-    expect(actual.numRows).toBe(3);
+          "struct",
-    const actualSchema = actual.schema;
+          new Struct([
-    expect(actualSchema).toEqual(schema);
+            new Field("x", new Float64(), false),
-  });
+            new Field("y", new Float64(), false),
-
+          ]),
-  it("will allow different vector column types", async function () {
-    const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], {
-      vectorColumns: {
-        fp16: { type: new Float16() },
-        fp32: { type: new Float32() },
-        fp64: { type: new Float64() },
-      },
-    });
-
-    expect(table.getChild("fp16")?.type.children[0].type.toString()).toEqual(
-      new Float16().toString(),
-    );
-    expect(table.getChild("fp32")?.type.children[0].type.toString()).toEqual(
-      new Float32().toString(),
-    );
-    expect(table.getChild("fp64")?.type.children[0].type.toString()).toEqual(
-      new Float64().toString(),
-    );
-  });
-
-  it("will use dictionary encoded strings if asked", async function () {
-    const table = makeArrowTable([{ str: "hello" }]);
-    expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true);
-
-    const tableWithDict = makeArrowTable([{ str: "hello" }], {
-      dictionaryEncodeStrings: true,
-    });
-    expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe(
-      true,
-    );
-
-    const schema = new Schema([
-      new Field("str", new Dictionary(new Utf8(), new Int32())),
-    ]);
-
-    const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema });
-    expect(DataType.isDictionary(tableWithDict2.getChild("str")?.type)).toBe(
-      true,
-    );
-  });
-
-  it("will infer data types correctly", async function () {
-    await checkTableCreation(async (records) => makeArrowTable(records), true);
-  });
-
-  it("will allow a schema to be provided", async function () {
-    await checkTableCreation(
-      async (records, _, schema) => makeArrowTable(records, { schema }),
-      false,
-    );
-  });
-
-  it("will use the field order of any provided schema", async function () {
-    await checkTableCreation(
-      async (_, recordsReversed, schema) =>
-        makeArrowTable(recordsReversed, { schema }),
-      false,
-    );
-  });
-
-  it("will make an empty table", async function () {
-    await checkTableCreation(
-      async (_, __, schema) => makeArrowTable([], { schema }),
-      false,
-    );
-  });
-});
-
-class DummyEmbedding extends EmbeddingFunction<string> {
-  toJSON(): Partial<FunctionOptions> {
-    return {};
-  }
-
-  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
-    return data.map(() => [0.0, 0.0]);
-  }
-
-  ndims(): number {
-    return 2;
-  }
-
-  embeddingDataType() {
-    return new Float16();
-  }
-}
-
-class DummyEmbeddingWithNoDimension extends EmbeddingFunction<string> {
-  toJSON(): Partial<FunctionOptions> {
-    return {};
-  }
-
-  embeddingDataType(): Float {
-    return new Float16();
-  }
-
-  async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
-    return data.map(() => [0.0, 0.0]);
-  }
-}
-const dummyEmbeddingConfig: EmbeddingFunctionConfig = {
-  sourceColumn: "string",
-  function: new DummyEmbedding(),
-};
-
-const dummyEmbeddingConfigWithNoDimension: EmbeddingFunctionConfig = {
-  sourceColumn: "string",
-  function: new DummyEmbeddingWithNoDimension(),
-};
-
-describe("convertToTable", function () {
-  it("will infer data types correctly", async function () {
-    await checkTableCreation(
-      async (records) => await convertToTable(records),
-      true,
-    );
-  });
-
-  it("will allow a schema to be provided", async function () {
-    await checkTableCreation(
-      async (records, _, schema) =>
-        await convertToTable(records, undefined, { schema }),
-      false,
-    );
-  });
-
-  it("will use the field order of any provided schema", async function () {
-    await checkTableCreation(
-      async (_, recordsReversed, schema) =>
-        await convertToTable(recordsReversed, undefined, { schema }),
-      false,
-    );
-  });
-
-  it("will make an empty table", async function () {
-    await checkTableCreation(
-      async (_, __, schema) => await convertToTable([], undefined, { schema }),
-      false,
-    );
-  });
-
-  it("will apply embeddings", async function () {
-    const records = sampleRecords();
-    const table = await convertToTable(records, dummyEmbeddingConfig);
-    expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
-    expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
-      new Float16().toString(),
-    );
-  });
-
-  it("will fail if missing the embedding source column", async function () {
-    await expect(
-      convertToTable([{ id: 1 }], dummyEmbeddingConfig),
-    ).rejects.toThrow("'string' was not present");
-  });
-
-  it("use embeddingDimension if embedding missing from table", async function () {
-    const schema = new Schema([new Field("string", new Utf8(), false)]);
-    // Simulate getting an empty Arrow table (minus embedding) from some other source
-    // In other words, we aren't starting with records
-    const table = makeEmptyTable(schema);
-
-    // If the embedding specifies the dimension we are fine
-    await fromTableToBuffer(table, dummyEmbeddingConfig);
-
-    // We can also supply a schema and should be ok
-    const schemaWithEmbedding = new Schema([
-      new Field("string", new Utf8(), false),
-      new Field(
-        "vector",
-        new FixedSizeList(2, new Field("item", new Float16(), false)),
-        false,
-      ),
-    ]);
-    await fromTableToBuffer(
-      table,
-      dummyEmbeddingConfigWithNoDimension,
-      schemaWithEmbedding,
-    );
-
-    // Otherwise we will get an error
-    await expect(
-      fromTableToBuffer(table, dummyEmbeddingConfigWithNoDimension),
-    ).rejects.toThrow("does not specify `embeddingDimension`");
-  });
-
-  it("will apply embeddings to an empty table", async function () {
-    const schema = new Schema([
-      new Field("string", new Utf8(), false),
-      new Field(
-        "vector",
-        new FixedSizeList(2, new Field("item", new Float16(), false)),
-        false,
-      ),
-    ]);
-    const table = await convertToTable([], dummyEmbeddingConfig, { schema });
-    expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(true);
-    expect(table.getChild("vector")?.type.children[0].type.toString()).toEqual(
-      new Float16().toString(),
-    );
-  });
-
-  it("will complain if embeddings present but schema missing embedding column", async function () {
-    const schema = new Schema([new Field("string", new Utf8(), false)]);
-    await expect(
-      convertToTable([], dummyEmbeddingConfig, { schema }),
-    ).rejects.toThrow("column vector was missing");
-  });
-
-  it("will provide a nice error if run twice", async function () {
-    const records = sampleRecords();
-    const table = await convertToTable(records, dummyEmbeddingConfig);
-
-    // fromTableToBuffer will try and apply the embeddings again
-    await expect(
-      fromTableToBuffer(table, dummyEmbeddingConfig),
-    ).rejects.toThrow("already existed");
-  });
-});
-
-describe("makeEmptyTable", function () {
-  it("will make an empty table", async function () {
-    await checkTableCreation(
-      async (_, __, schema) => makeEmptyTable(schema),
-      false,
-    );
-  });
-});
-
-describe("when using two versions of arrow", function () {
-  it("can still import data", async function () {
-    const schema = new OldSchema([
-      new OldField("id", new OldInt32()),
-      new OldField(
-        "vector",
-        new OldFixedSizeList(
-          1024,
-          new OldField("item", new OldFloat32(), true),
         ),
-      ),
+        new Field(
-      new OldField(
+          "list",
-        "struct",
+          new List(new Field("item", new Utf8(), false)),
-        new OldStruct([
+          false,
-          new OldField(
+        ),
-            "nested",
+      ]);
-            new OldDictionary(new OldUtf8(), new OldInt32(), 1, true),
+
+      const table = (await tableCreationMethod(
+        records,
+        recordsReversed,
+        schema,
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+      )) as any;
+      schema.fields.forEach(
+        (
+          // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+          field: { name: any; type: { toString: () => any } },
+          idx: string | number,
+        ) => {
+          const actualField = table.schema.fields[idx];
+          // Type inference always assumes nullable=true
+          if (infersTypes) {
+            expect(actualField.nullable).toBe(true);
+          } else {
+            expect(actualField.nullable).toBe(false);
+          }
+          expect(table.getChild(field.name)?.type.toString()).toEqual(
+            field.type.toString(),
+          );
+          expect(table.getChildAt(idx)?.type.toString()).toEqual(
+            field.type.toString(),
+          );
+        },
+      );
+    }
+
+    describe("The function makeArrowTable", function () {
+      it("will use data types from a provided schema instead of inference", async function () {
+        const schema = new Schema([
+          new Field("a", new Int32()),
+          new Field("b", new Float32()),
+          new Field(
+            "c",
+            new FixedSizeList(3, new Field("item", new Float16())),
           ),
-          new OldField("ts_with_tz", new OldTimestampNanosecond("some_tz")),
+          new Field("d", new Int64()),
-          new OldField("ts_no_tz", new OldTimestampNanosecond(null)),
+        ]);
-        ]),
+        const table = makeArrowTable(
-      ),
+          [
-      // biome-ignore lint/suspicious/noExplicitAny: skip
+            { a: 1, b: 2, c: [1, 2, 3], d: 9 },
-    ]) as any;
+            { a: 4, b: 5, c: [4, 5, 6], d: 10 },
-    schema.metadataVersion = MetadataVersion.V5;
+            { a: 7, b: 8, c: [7, 8, 9], d: null },
-    const table = makeArrowTable([], { schema });
+          ],
+          { schema },
+        );
 
-    const buf = await fromTableToBuffer(table);
+        const buf = await fromTableToBuffer(table);
-    expect(buf.byteLength).toBeGreaterThan(0);
+        expect(buf.byteLength).toBeGreaterThan(0);
-    const actual = tableFromIPC(buf);
-    const actualSchema = actual.schema;
-    expect(actualSchema.fields.length).toBe(3);
 
-    // Deep equality gets hung up on some very minor unimportant differences
+        const actual = tableFromIPC(buf);
-    // between arrow version 13 and 15 which isn't really what we're testing for
+        expect(actual.numRows).toBe(3);
-    // and so we do our own comparison that just checks name/type/nullability
+        const actualSchema = actual.schema;
-    function compareFields(lhs: Field, rhs: Field) {
+        expect(actualSchema).toEqual(schema);
-      expect(lhs.name).toEqual(rhs.name);
+      });
-      expect(lhs.nullable).toEqual(rhs.nullable);
+
-      expect(lhs.typeId).toEqual(rhs.typeId);
+      it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
-      if ("children" in lhs.type && lhs.type.children !== null) {
+        const schema = new Schema([
-        const lhsChildren = lhs.type.children as Field[];
+          new Field("a", new Float(Precision.DOUBLE), true),
-        lhsChildren.forEach((child: Field, idx) => {
+          new Field("b", new Float(Precision.DOUBLE), true),
-          compareFields(child, rhs.type.children[idx]);
+          new Field(
+            "vector",
+            new FixedSizeList(
+              3,
+              new Field("item", new Float(Precision.SINGLE), true),
+            ),
+            true,
+          ),
+        ]);
+        const table = makeArrowTable([
+          { a: 1, b: 2, vector: [1, 2, 3] },
+          { a: 4, b: 5, vector: [4, 5, 6] },
+          { a: 7, b: 8, vector: [7, 8, 9] },
+        ]);
+
+        const buf = await fromTableToBuffer(table);
+        expect(buf.byteLength).toBeGreaterThan(0);
+
+        const actual = tableFromIPC(buf);
+        expect(actual.numRows).toBe(3);
+        const actualSchema = actual.schema;
+        expect(actualSchema).toEqual(schema);
+      });
+
+      it("can support multiple vector columns", async function () {
+        const schema = new Schema([
+          new Field("a", new Float(Precision.DOUBLE), true),
+          new Field("b", new Float(Precision.DOUBLE), true),
+          new Field(
+            "vec1",
+            new FixedSizeList(3, new Field("item", new Float16(), true)),
+            true,
+          ),
+          new Field(
+            "vec2",
+            new FixedSizeList(3, new Field("item", new Float16(), true)),
+            true,
+          ),
+        ]);
+        const table = makeArrowTable(
+          [
+            { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+            { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+            { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] },
+          ],
+          {
+            vectorColumns: {
+              vec1: { type: new Float16() },
+              vec2: { type: new Float16() },
+            },
+          },
+        );
+
+        const buf = await fromTableToBuffer(table);
+        expect(buf.byteLength).toBeGreaterThan(0);
+
+        const actual = tableFromIPC(buf);
+        expect(actual.numRows).toBe(3);
+        const actualSchema = actual.schema;
+        expect(actualSchema).toEqual(schema);
+      });
+
+      it("will allow different vector column types", async function () {
+        const table = makeArrowTable([{ fp16: [1], fp32: [1], fp64: [1] }], {
+          vectorColumns: {
+            fp16: { type: new Float16() },
+            fp32: { type: new Float32() },
+            fp64: { type: new Float64() },
+          },
         });
+
+        expect(
+          table.getChild("fp16")?.type.children[0].type.toString(),
+        ).toEqual(new Float16().toString());
+        expect(
+          table.getChild("fp32")?.type.children[0].type.toString(),
+        ).toEqual(new Float32().toString());
+        expect(
+          table.getChild("fp64")?.type.children[0].type.toString(),
+        ).toEqual(new Float64().toString());
+      });
+
+      it("will use dictionary encoded strings if asked", async function () {
+        const table = makeArrowTable([{ str: "hello" }]);
+        expect(DataType.isUtf8(table.getChild("str")?.type)).toBe(true);
+
+        const tableWithDict = makeArrowTable([{ str: "hello" }], {
+          dictionaryEncodeStrings: true,
+        });
+        expect(DataType.isDictionary(tableWithDict.getChild("str")?.type)).toBe(
+          true,
+        );
+
+        const schema = new Schema([
+          new Field("str", new Dictionary(new Utf8(), new Int32())),
+        ]);
+
+        const tableWithDict2 = makeArrowTable([{ str: "hello" }], { schema });
+        expect(
+          DataType.isDictionary(tableWithDict2.getChild("str")?.type),
+        ).toBe(true);
+      });
+
+      it("will infer data types correctly", async function () {
+        await checkTableCreation(
+          // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+          async (records) => (<any>makeArrowTable)(records),
+          true,
+        );
+      });
+
+      it("will allow a schema to be provided", async function () {
+        await checkTableCreation(
+          async (records, _, schema) =>
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            (<any>makeArrowTable)(records, { schema }),
+          false,
+        );
+      });
+
+      it("will use the field order of any provided schema", async function () {
+        await checkTableCreation(
+          async (_, recordsReversed, schema) =>
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            (<any>makeArrowTable)(recordsReversed, { schema }),
+          false,
+        );
+      });
+
+      it("will make an empty table", async function () {
+        await checkTableCreation(
+          // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+          async (_, __, schema) => (<any>makeArrowTable)([], { schema }),
+          false,
+        );
+      });
+    });
+
+    class DummyEmbedding extends EmbeddingFunction<string> {
+      toJSON(): Partial<FunctionOptions> {
+        return {};
+      }
+
+      async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
+        return data.map(() => [0.0, 0.0]);
+      }
+
+      ndims(): number {
+        return 2;
+      }
+
+      embeddingDataType() {
+        return new Float16();
       }
     }
-    actualSchema.fields.forEach((field, idx) => {
+
-      compareFields(field, actualSchema.fields[idx]);
+    class DummyEmbeddingWithNoDimension extends EmbeddingFunction<string> {
+      toJSON(): Partial<FunctionOptions> {
+        return {};
+      }
+
+      embeddingDataType() {
+        return new Float16();
+      }
+
+      async computeSourceEmbeddings(data: string[]): Promise<number[][]> {
+        return data.map(() => [0.0, 0.0]);
+      }
+    }
+    const dummyEmbeddingConfig: EmbeddingFunctionConfig = {
+      sourceColumn: "string",
+      function: new DummyEmbedding(),
+    };
+
+    const dummyEmbeddingConfigWithNoDimension: EmbeddingFunctionConfig = {
+      sourceColumn: "string",
+      function: new DummyEmbeddingWithNoDimension(),
+    };
+
+    describe("convertToTable", function () {
+      it("will infer data types correctly", async function () {
+        await checkTableCreation(
+          // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+          async (records) => await (<any>convertToTable)(records),
+          true,
+        );
+      });
+
+      it("will allow a schema to be provided", async function () {
+        await checkTableCreation(
+          async (records, _, schema) =>
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            await (<any>convertToTable)(records, undefined, { schema }),
+          false,
+        );
+      });
+
+      it("will use the field order of any provided schema", async function () {
+        await checkTableCreation(
+          async (_, recordsReversed, schema) =>
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            await (<any>convertToTable)(recordsReversed, undefined, { schema }),
+          false,
+        );
+      });
+
+      it("will make an empty table", async function () {
+        await checkTableCreation(
+          async (_, __, schema) =>
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            await (<any>convertToTable)([], undefined, { schema }),
+          false,
+        );
+      });
+
+      it("will apply embeddings", async function () {
+        const records = sampleRecords();
+        const table = await convertToTable(records, dummyEmbeddingConfig);
+        expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(
+          true,
+        );
+        expect(
+          table.getChild("vector")?.type.children[0].type.toString(),
+        ).toEqual(new Float16().toString());
+      });
+
+      it("will fail if missing the embedding source column", async function () {
+        await expect(
+          convertToTable([{ id: 1 }], dummyEmbeddingConfig),
+        ).rejects.toThrow("'string' was not present");
+      });
+
+      it("use embeddingDimension if embedding missing from table", async function () {
+        const schema = new Schema([new Field("string", new Utf8(), false)]);
+        // Simulate getting an empty Arrow table (minus embedding) from some other source
+        // In other words, we aren't starting with records
+        const table = makeEmptyTable(schema);
+
+        // If the embedding specifies the dimension we are fine
+        await fromTableToBuffer(table, dummyEmbeddingConfig);
+
+        // We can also supply a schema and should be ok
+        const schemaWithEmbedding = new Schema([
+          new Field("string", new Utf8(), false),
+          new Field(
+            "vector",
+            new FixedSizeList(2, new Field("item", new Float16(), false)),
+            false,
+          ),
+        ]);
+        await fromTableToBuffer(
+          table,
+          dummyEmbeddingConfigWithNoDimension,
+          schemaWithEmbedding,
+        );
+
+        // Otherwise we will get an error
+        await expect(
+          fromTableToBuffer(table, dummyEmbeddingConfigWithNoDimension),
+        ).rejects.toThrow("does not specify `embeddingDimension`");
+      });
+
+      it("will apply embeddings to an empty table", async function () {
+        const schema = new Schema([
+          new Field("string", new Utf8(), false),
+          new Field(
+            "vector",
+            new FixedSizeList(2, new Field("item", new Float16(), false)),
+            false,
+          ),
+        ]);
+        const table = await convertToTable([], dummyEmbeddingConfig, {
+          schema,
+        });
+        expect(DataType.isFixedSizeList(table.getChild("vector")?.type)).toBe(
+          true,
+        );
+        expect(
+          table.getChild("vector")?.type.children[0].type.toString(),
+        ).toEqual(new Float16().toString());
+      });
+
+      it("will complain if embeddings present but schema missing embedding column", async function () {
+        const schema = new Schema([new Field("string", new Utf8(), false)]);
+        await expect(
+          convertToTable([], dummyEmbeddingConfig, { schema }),
+        ).rejects.toThrow("column vector was missing");
+      });
+
+      it("will provide a nice error if run twice", async function () {
+        const records = sampleRecords();
+        const table = await convertToTable(records, dummyEmbeddingConfig);
+
+        // fromTableToBuffer will try and apply the embeddings again
+        await expect(
+          fromTableToBuffer(table, dummyEmbeddingConfig),
+        ).rejects.toThrow("already existed");
+      });
     });
-  });
+
-});
+    describe("makeEmptyTable", function () {
+      it("will make an empty table", async function () {
+        await checkTableCreation(
+          // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+          async (_, __, schema) => (<any>makeEmptyTable)(schema),
+          false,
+        );
+      });
+    });
+
+    describe("when using two versions of arrow", function () {
+      it("can still import data", async function () {
+        const schema = new arrow13.Schema([
+          new arrow13.Field("id", new arrow13.Int32()),
+          new arrow13.Field(
+            "vector",
+            new arrow13.FixedSizeList(
+              1024,
+              new arrow13.Field("item", new arrow13.Float32(), true),
+            ),
+          ),
+          new arrow13.Field(
+            "struct",
+            new arrow13.Struct([
+              new arrow13.Field(
+                "nested",
+                new arrow13.Dictionary(
+                  new arrow13.Utf8(),
+                  new arrow13.Int32(),
+                  1,
+                  true,
+                ),
+              ),
+              new arrow13.Field(
+                "ts_with_tz",
+                new arrow13.TimestampNanosecond("some_tz"),
+              ),
+              new arrow13.Field(
+                "ts_no_tz",
+                new arrow13.TimestampNanosecond(null),
+              ),
+            ]),
+          ),
+          // biome-ignore lint/suspicious/noExplicitAny: skip
+        ]) as any;
+        schema.metadataVersion = arrow13.MetadataVersion.V5;
+        const table = makeArrowTable([], { schema });
+
+        const buf = await fromTableToBuffer(table);
+        expect(buf.byteLength).toBeGreaterThan(0);
+        const actual = tableFromIPC(buf);
+        const actualSchema = actual.schema;
+        expect(actualSchema.fields.length).toBe(3);
+
+        // Deep equality gets hung up on some very minor unimportant differences
+        // between arrow version 13 and 15 which isn't really what we're testing for
+        // and so we do our own comparison that just checks name/type/nullability
+        function compareFields(lhs: arrow13.Field, rhs: arrow13.Field) {
+          expect(lhs.name).toEqual(rhs.name);
+          expect(lhs.nullable).toEqual(rhs.nullable);
+          expect(lhs.typeId).toEqual(rhs.typeId);
+          if ("children" in lhs.type && lhs.type.children !== null) {
+            const lhsChildren = lhs.type.children as arrow13.Field[];
+            lhsChildren.forEach((child: arrow13.Field, idx) => {
+              compareFields(child, rhs.type.children[idx]);
+            });
+          }
+        }
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        actualSchema.fields.forEach((field: any, idx: string | number) => {
+          compareFields(field, actualSchema.fields[idx]);
+        });
+      });
+    });
+  },
+);

nodejs/__test__/registry.test.ts

@@ -11,8 +11,11 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-import * as arrow from "apache-arrow";
+import * as arrow13 from "apache-arrow-13";
-import * as arrowOld from "apache-arrow-old";
+import * as arrow14 from "apache-arrow-14";
+import * as arrow15 from "apache-arrow-15";
+import * as arrow16 from "apache-arrow-16";
+import * as arrow17 from "apache-arrow-17";
 
 import * as tmp from "tmp";
 
@@ -20,150 +23,154 @@ import { connect } from "../lancedb";
 import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
 import { getRegistry, register } from "../lancedb/embedding/registry";
 
-describe.each([arrow, arrowOld])("LanceSchema", (arrow) => {
+describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
-  test("should preserve input order", async () => {
+  "LanceSchema",
-    const schema = LanceSchema({
+  (arrow) => {
-      id: new arrow.Int32(),
+    test("should preserve input order", async () => {
-      text: new arrow.Utf8(),
+      const schema = LanceSchema({
-      vector: new arrow.Float32(),
+        id: new arrow.Int32(),
-    });
+        text: new arrow.Utf8(),
-    expect(schema.fields.map((x) => x.name)).toEqual(["id", "text", "vector"]);
+        vector: new arrow.Float32(),
-  });
-});
-
-describe("Registry", () => {
-  let tmpDir: tmp.DirResult;
-  beforeEach(() => {
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
-  });
-
-  afterEach(() => {
-    tmpDir.removeCallback();
-    getRegistry().reset();
-  });
-
-  it("should register a new item to the registry", async () => {
-    @register("mock-embedding")
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
-      toJSON(): object {
-        return {
-          someText: "hello",
-        };
-      }
-      constructor() {
-        super();
-      }
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): arrow.Float {
-        return new arrow.Float32();
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        return data.map(() => [1, 2, 3]);
-      }
-    }
-    const func = getRegistry()
-      .get<MockEmbeddingFunction>("mock-embedding")!
-      .create();
-
-    const schema = LanceSchema({
-      id: new arrow.Int32(),
-      text: func.sourceField(new arrow.Utf8()),
-      vector: func.vectorField(),
-    });
-
-    const db = await connect(tmpDir.name);
-    const table = await db.createTable(
-      "test",
-      [
-        { id: 1, text: "hello" },
-        { id: 2, text: "world" },
-      ],
-      { schema },
-    );
-    const expected = [
-      [1, 2, 3],
-      [1, 2, 3],
-    ];
-    const actual = await table.query().toArrow();
-    const vectors = actual
-      .getChild("vector")
-      ?.toArray()
-      .map((x: unknown) => {
-        if (x instanceof arrow.Vector) {
-          return [...x];
-        } else {
-          return x;
-        }
       });
-    expect(vectors).toEqual(expected);
+      expect(schema.fields.map((x) => x.name)).toEqual([
-  });
+        "id",
-  test("should error if registering with the same name", async () => {
+        "text",
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+        "vector",
-      toJSON(): object {
+      ]);
-        return {
-          someText: "hello",
-        };
-      }
-      constructor() {
-        super();
-      }
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): arrow.Float {
-        return new arrow.Float32();
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        return data.map(() => [1, 2, 3]);
-      }
-    }
-    register("mock-embedding")(MockEmbeddingFunction);
-    expect(() => register("mock-embedding")(MockEmbeddingFunction)).toThrow(
-      'Embedding function with alias "mock-embedding" already exists',
-    );
-  });
-  test("schema should contain correct metadata", async () => {
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
-      toJSON(): object {
-        return {
-          someText: "hello",
-        };
-      }
-      constructor() {
-        super();
-      }
-      ndims() {
-        return 3;
-      }
-      embeddingDataType(): arrow.Float {
-        return new arrow.Float32();
-      }
-      async computeSourceEmbeddings(data: string[]) {
-        return data.map(() => [1, 2, 3]);
-      }
-    }
-    const func = new MockEmbeddingFunction();
-
-    const schema = LanceSchema({
-      id: new arrow.Int32(),
-      text: func.sourceField(new arrow.Utf8()),
-      vector: func.vectorField(),
     });
-    const expectedMetadata = new Map<string, string>([
+  },
-      [
+);
-        "embedding_functions",
+
-        JSON.stringify([
+describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
-          {
+  "Registry",
-            sourceColumn: "text",
+  (arrow) => {
-            vectorColumn: "vector",
+    let tmpDir: tmp.DirResult;
-            name: "MockEmbeddingFunction",
+    beforeEach(() => {
-            model: { someText: "hello" },
+      tmpDir = tmp.dirSync({ unsafeCleanup: true });
-          },
+    });
-        ]),
+
-      ],
+    afterEach(() => {
-    ]);
+      tmpDir.removeCallback();
-    expect(schema.metadata).toEqual(expectedMetadata);
+      getRegistry().reset();
-  });
+    });
-});
+
+    it("should register a new item to the registry", async () => {
+      @register("mock-embedding")
+      class MockEmbeddingFunction extends EmbeddingFunction<string> {
+        toJSON(): object {
+          return {
+            someText: "hello",
+          };
+        }
+        constructor() {
+          super();
+        }
+        ndims() {
+          return 3;
+        }
+        embeddingDataType() {
+          return new arrow.Float32();
+        }
+        async computeSourceEmbeddings(data: string[]) {
+          return data.map(() => [1, 2, 3]);
+        }
+      }
+
+      const func = getRegistry()
+        .get<MockEmbeddingFunction>("mock-embedding")!
+        .create();
+
+      const schema = LanceSchema({
+        id: new arrow.Int32(),
+        text: func.sourceField(new arrow.Utf8()),
+        vector: func.vectorField(),
+      });
+
+      const db = await connect(tmpDir.name);
+      const table = await db.createTable(
+        "test",
+        [
+          { id: 1, text: "hello" },
+          { id: 2, text: "world" },
+        ],
+        { schema },
+      );
+      const expected = [
+        [1, 2, 3],
+        [1, 2, 3],
+      ];
+      const actual = await table.query().toArrow();
+      const vectors = actual.getChild("vector")!.toArray();
+      expect(JSON.parse(JSON.stringify(vectors))).toEqual(
+        JSON.parse(JSON.stringify(expected)),
+      );
+    });
+    test("should error if registering with the same name", async () => {
+      class MockEmbeddingFunction extends EmbeddingFunction<string> {
+        toJSON(): object {
+          return {
+            someText: "hello",
+          };
+        }
+        constructor() {
+          super();
+        }
+        ndims() {
+          return 3;
+        }
+        embeddingDataType() {
+          return new arrow.Float32();
+        }
+        async computeSourceEmbeddings(data: string[]) {
+          return data.map(() => [1, 2, 3]);
+        }
+      }
+      register("mock-embedding")(MockEmbeddingFunction);
+      expect(() => register("mock-embedding")(MockEmbeddingFunction)).toThrow(
+        'Embedding function with alias "mock-embedding" already exists',
+      );
+    });
+    test("schema should contain correct metadata", async () => {
+      class MockEmbeddingFunction extends EmbeddingFunction<string> {
+        toJSON(): object {
+          return {
+            someText: "hello",
+          };
+        }
+        constructor() {
+          super();
+        }
+        ndims() {
+          return 3;
+        }
+        embeddingDataType() {
+          return new arrow.Float32();
+        }
+        async computeSourceEmbeddings(data: string[]) {
+          return data.map(() => [1, 2, 3]);
+        }
+      }
+      const func = new MockEmbeddingFunction();
+
+      const schema = LanceSchema({
+        id: new arrow.Int32(),
+        text: func.sourceField(new arrow.Utf8()),
+        vector: func.vectorField(),
+      });
+      const expectedMetadata = new Map<string, string>([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              sourceColumn: "text",
+              vectorColumn: "vector",
+              name: "MockEmbeddingFunction",
+              model: { someText: "hello" },
+            },
+          ]),
+        ],
+      ]);
+      expect(schema.metadata).toEqual(expectedMetadata);
+    });
+  },
+);

nodejs/__test__/s3_integration.test.ts

@@ -14,6 +14,11 @@
 
 /* eslint-disable @typescript-eslint/naming-convention */
 
+import {
+  CreateTableCommand,
+  DeleteTableCommand,
+  DynamoDBClient,
+} from "@aws-sdk/client-dynamodb";
 import {
   CreateKeyCommand,
   KMSClient,
@@ -38,6 +43,7 @@ const CONFIG = {
   awsAccessKeyId: "ACCESSKEY",
   awsSecretAccessKey: "SECRETKEY",
   awsEndpoint: "http://127.0.0.1:4566",
+  dynamodbEndpoint: "http://127.0.0.1:4566",
   awsRegion: "us-east-1",
 };
 
@@ -66,7 +72,6 @@ class S3Bucket {
     } catch {
       // It's fine if the bucket doesn't exist
     }
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new CreateBucketCommand({ Bucket: name }));
     return new S3Bucket(name);
   }
@@ -79,32 +84,27 @@ class S3Bucket {
   static async deleteBucket(client: S3Client, name: string) {
     // Must delete all objects before we can delete the bucket
     const objects = await client.send(
-      // biome-ignore lint/style/useNamingConvention: we dont control s3's api
       new ListObjectsV2Command({ Bucket: name }),
     );
     if (objects.Contents) {
       for (const object of objects.Contents) {
         await client.send(
-          // biome-ignore lint/style/useNamingConvention: we dont control s3's api
           new DeleteObjectCommand({ Bucket: name, Key: object.Key }),
         );
       }
     }
 
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new DeleteBucketCommand({ Bucket: name }));
   }
 
   public async assertAllEncrypted(path: string, keyId: string) {
     const client = S3Bucket.s3Client();
     const objects = await client.send(
-      // biome-ignore lint/style/useNamingConvention: we dont control s3's api
       new ListObjectsV2Command({ Bucket: this.name, Prefix: path }),
     );
     if (objects.Contents) {
       for (const object of objects.Contents) {
         const metadata = await client.send(
-          // biome-ignore lint/style/useNamingConvention: we dont control s3's api
           new HeadObjectCommand({ Bucket: this.name, Key: object.Key }),
         );
         expect(metadata.ServerSideEncryption).toBe("aws:kms");
@@ -143,7 +143,6 @@ class KmsKey {
 
   public async delete() {
     const client = KmsKey.kmsClient();
-    // biome-ignore lint/style/useNamingConvention: we dont control s3's api
     await client.send(new ScheduleKeyDeletionCommand({ KeyId: this.keyId }));
   }
 }
@@ -224,3 +223,91 @@ maybeDescribe("storage_options", () => {
     await bucket.assertAllEncrypted("test/table2.lance", kmsKey.keyId);
   });
 });
+
+class DynamoDBCommitTable {
+  name: string;
+  constructor(name: string) {
+    this.name = name;
+  }
+
+  static dynamoClient() {
+    return new DynamoDBClient({
+      region: CONFIG.awsRegion,
+      credentials: {
+        accessKeyId: CONFIG.awsAccessKeyId,
+        secretAccessKey: CONFIG.awsSecretAccessKey,
+      },
+      endpoint: CONFIG.awsEndpoint,
+    });
+  }
+
+  public static async create(name: string): Promise<DynamoDBCommitTable> {
+    const client = DynamoDBCommitTable.dynamoClient();
+    const command = new CreateTableCommand({
+      TableName: name,
+      AttributeDefinitions: [
+        {
+          AttributeName: "base_uri",
+          AttributeType: "S",
+        },
+        {
+          AttributeName: "version",
+          AttributeType: "N",
+        },
+      ],
+      KeySchema: [
+        { AttributeName: "base_uri", KeyType: "HASH" },
+        { AttributeName: "version", KeyType: "RANGE" },
+      ],
+      ProvisionedThroughput: {
+        ReadCapacityUnits: 1,
+        WriteCapacityUnits: 1,
+      },
+    });
+    await client.send(command);
+    return new DynamoDBCommitTable(name);
+  }
+
+  public async delete() {
+    const client = DynamoDBCommitTable.dynamoClient();
+    await client.send(new DeleteTableCommand({ TableName: this.name }));
+  }
+}
+
+maybeDescribe("DynamoDB Lock", () => {
+  let bucket: S3Bucket;
+  let commitTable: DynamoDBCommitTable;
+
+  beforeAll(async () => {
+    bucket = await S3Bucket.create("lancedb2");
+    commitTable = await DynamoDBCommitTable.create("commitTable");
+  });
+
+  afterAll(async () => {
+    await commitTable.delete();
+    await bucket.delete();
+  });
+
+  it("can be used to configure a DynamoDB table for commit log", async () => {
+    const uri = `s3+ddb://${bucket.name}/test?ddbTableName=${commitTable.name}`;
+    const db = await connect(uri, {
+      storageOptions: CONFIG,
+      readConsistencyInterval: 0,
+    });
+
+    const table = await db.createTable("test", [{ a: 1, b: 2 }]);
+
+    // 5 concurrent appends
+    const futs = Array.from({ length: 5 }, async () => {
+      // Open a table so each append has a separate table reference. Otherwise
+      // they will share the same table reference and the internal ReadWriteLock
+      // will prevent any real concurrency.
+      const table = await db.openTable("test");
+      await table.add([{ a: 2, b: 3 }]);
+    });
+    await Promise.all(futs);
+
+    const rowCount = await table.countRows();
+    expect(rowCount).toBe(6);
+  });
+});

nodejs/__test__/table.test.ts

@@ -16,8 +16,11 @@ import * as fs from "fs";
 import * as path from "path";
 import * as tmp from "tmp";
 
-import * as arrow from "apache-arrow";
+import * as arrow13 from "apache-arrow-13";
-import * as arrowOld from "apache-arrow-old";
+import * as arrow14 from "apache-arrow-14";
+import * as arrow15 from "apache-arrow-15";
+import * as arrow16 from "apache-arrow-16";
+import * as arrow17 from "apache-arrow-17";
 
 import { Table, connect } from "../lancedb";
 import {
@@ -31,106 +34,163 @@ import {
   Schema,
   makeArrowTable,
 } from "../lancedb/arrow";
-import { EmbeddingFunction, LanceSchema, register } from "../lancedb/embedding";
+import {
+  EmbeddingFunction,
+  LanceSchema,
+  getRegistry,
+  register,
+} from "../lancedb/embedding";
 import { Index } from "../lancedb/indices";
 
-// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
-describe.each([arrow, arrowOld])("Given a table", (arrow: any) => {
+  "Given a table",
-  let tmpDir: tmp.DirResult;
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  let table: Table;
+  (arrow: any) => {
+    let tmpDir: tmp.DirResult;
+    let table: Table;
 
-  const schema = new arrow.Schema([
+    const schema:
-    new arrow.Field("id", new arrow.Float64(), true),
+      | import("apache-arrow-13").Schema
-  ]);
+      | import("apache-arrow-14").Schema
+      | import("apache-arrow-15").Schema
+      | import("apache-arrow-16").Schema
+      | import("apache-arrow-17").Schema = new arrow.Schema([
+      new arrow.Field("id", new arrow.Float64(), true),
+    ]);
 
-  beforeEach(async () => {
+    beforeEach(async () => {
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+      tmpDir = tmp.dirSync({ unsafeCleanup: true });
-    const conn = await connect(tmpDir.name);
+      const conn = await connect(tmpDir.name);
-    table = await conn.createEmptyTable("some_table", schema);
+      table = await conn.createEmptyTable("some_table", schema);
-  });
-  afterEach(() => tmpDir.removeCallback());
-
-  it("be displayable", async () => {
-    expect(table.display()).toMatch(
-      /NativeTable\(some_table, uri=.*, read_consistency_interval=None\)/,
-    );
-    table.close();
-    expect(table.display()).toBe("ClosedTable(some_table)");
-  });
-
-  it("should let me add data", async () => {
-    await table.add([{ id: 1 }, { id: 2 }]);
-    await table.add([{ id: 1 }]);
-    await expect(table.countRows()).resolves.toBe(3);
-  });
-
-  it("should overwrite data if asked", async () => {
-    await table.add([{ id: 1 }, { id: 2 }]);
-    await table.add([{ id: 1 }], { mode: "overwrite" });
-    await expect(table.countRows()).resolves.toBe(1);
-  });
-
-  it("should let me close the table", async () => {
-    expect(table.isOpen()).toBe(true);
-    table.close();
-    expect(table.isOpen()).toBe(false);
-    expect(table.countRows()).rejects.toThrow("Table some_table is closed");
-  });
-
-  it("should let me update values", async () => {
-    await table.add([{ id: 1 }]);
-    expect(await table.countRows("id == 1")).toBe(1);
-    expect(await table.countRows("id == 7")).toBe(0);
-    await table.update({ id: "7" });
-    expect(await table.countRows("id == 1")).toBe(0);
-    expect(await table.countRows("id == 7")).toBe(1);
-    await table.add([{ id: 2 }]);
-    // Test Map as input
-    await table.update(new Map(Object.entries({ id: "10" })), {
-      where: "id % 2 == 0",
     });
-    expect(await table.countRows("id == 2")).toBe(0);
+    afterEach(() => tmpDir.removeCallback());
-    expect(await table.countRows("id == 7")).toBe(1);
-    expect(await table.countRows("id == 10")).toBe(1);
-  });
 
-  // https://github.com/lancedb/lancedb/issues/1293
+    it("be displayable", async () => {
-  test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
+      expect(table.display()).toMatch(
-    "can create empty table with non default float type: %s",
+        /NativeTable\(some_table, uri=.*, read_consistency_interval=None\)/,
-    async (floatType) => {
+      );
-      const db = await connect(tmpDir.name);
+      table.close();
+      expect(table.display()).toBe("ClosedTable(some_table)");
+    });
 
-      const data = [
+    it("should let me add data", async () => {
-        { text: "hello", vector: Array(512).fill(1.0) },
+      await table.add([{ id: 1 }, { id: 2 }]);
-        { text: "hello world", vector: Array(512).fill(1.0) },
+      await table.add([{ id: 1 }]);
-      ];
+      await expect(table.countRows()).resolves.toBe(3);
-      const f64Schema = new arrow.Schema([
+    });
-        new arrow.Field("text", new arrow.Utf8(), true),
-        new arrow.Field(
-          "vector",
-          new arrow.FixedSizeList(512, new arrow.Field("item", floatType)),
-          true,
-        ),
-      ]);
 
-      const f64Table = await db.createEmptyTable("f64", f64Schema, {
+    it("should overwrite data if asked", async () => {
-        mode: "overwrite",
+      await table.add([{ id: 1 }, { id: 2 }]);
+      await table.add([{ id: 1 }], { mode: "overwrite" });
+      await expect(table.countRows()).resolves.toBe(1);
+    });
+
+    it("should let me close the table", async () => {
+      expect(table.isOpen()).toBe(true);
+      table.close();
+      expect(table.isOpen()).toBe(false);
+      expect(table.countRows()).rejects.toThrow("Table some_table is closed");
+    });
+
+    it("should let me update values", async () => {
+      await table.add([{ id: 1 }]);
+      expect(await table.countRows("id == 1")).toBe(1);
+      expect(await table.countRows("id == 7")).toBe(0);
+      await table.update({ id: "7" });
+      expect(await table.countRows("id == 1")).toBe(0);
+      expect(await table.countRows("id == 7")).toBe(1);
+      await table.add([{ id: 2 }]);
+      // Test Map as input
+      await table.update(new Map(Object.entries({ id: "10" })), {
+        where: "id % 2 == 0",
       });
-      try {
+      expect(await table.countRows("id == 2")).toBe(0);
-        await f64Table.add(data);
+      expect(await table.countRows("id == 7")).toBe(1);
-        const res = await f64Table.query().toArray();
+      expect(await table.countRows("id == 10")).toBe(1);
-        expect(res.length).toBe(2);
+    });
-      } catch (e) {
-        expect(e).toBeUndefined();
-      }
-    },
-  );
 
-  it("should return the table as an instance of an arrow table", async () => {
+    it("should let me update values with `values`", async () => {
-    const arrowTbl = await table.toArrow();
+      await table.add([{ id: 1 }]);
-    expect(arrowTbl).toBeInstanceOf(ArrowTable);
+      expect(await table.countRows("id == 1")).toBe(1);
-  });
+      expect(await table.countRows("id == 7")).toBe(0);
-});
+      await table.update({ values: { id: 7 } });
+      expect(await table.countRows("id == 1")).toBe(0);
+      expect(await table.countRows("id == 7")).toBe(1);
+      await table.add([{ id: 2 }]);
+      // Test Map as input
+      await table.update({
+        values: {
+          id: "10",
+        },
+        where: "id % 2 == 0",
+      });
+      expect(await table.countRows("id == 2")).toBe(0);
+      expect(await table.countRows("id == 7")).toBe(1);
+      expect(await table.countRows("id == 10")).toBe(1);
+    });
+
+    it("should let me update values with `valuesSql`", async () => {
+      await table.add([{ id: 1 }]);
+      expect(await table.countRows("id == 1")).toBe(1);
+      expect(await table.countRows("id == 7")).toBe(0);
+      await table.update({
+        valuesSql: {
+          id: "7",
+        },
+      });
+      expect(await table.countRows("id == 1")).toBe(0);
+      expect(await table.countRows("id == 7")).toBe(1);
+      await table.add([{ id: 2 }]);
+      // Test Map as input
+      await table.update({
+        valuesSql: {
+          id: "10",
+        },
+        where: "id % 2 == 0",
+      });
+      expect(await table.countRows("id == 2")).toBe(0);
+      expect(await table.countRows("id == 7")).toBe(1);
+      expect(await table.countRows("id == 10")).toBe(1);
+    });
+
+    // https://github.com/lancedb/lancedb/issues/1293
+    test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
+      "can create empty table with non default float type: %s",
+      async (floatType) => {
+        const db = await connect(tmpDir.name);
+
+        const data = [
+          { text: "hello", vector: Array(512).fill(1.0) },
+          { text: "hello world", vector: Array(512).fill(1.0) },
+        ];
+        const f64Schema = new arrow.Schema([
+          new arrow.Field("text", new arrow.Utf8(), true),
+          new arrow.Field(
+            "vector",
+            new arrow.FixedSizeList(512, new arrow.Field("item", floatType)),
+            true,
+          ),
+        ]);
+
+        const f64Table = await db.createEmptyTable("f64", f64Schema, {
+          mode: "overwrite",
+        });
+        try {
+          await f64Table.add(data);
+          const res = await f64Table.query().toArray();
+          expect(res.length).toBe(2);
+        } catch (e) {
+          expect(e).toBeUndefined();
+        }
+      },
+    );
+
+    it("should return the table as an instance of an arrow table", async () => {
+      const arrowTbl = await table.toArrow();
+      expect(arrowTbl).toBeInstanceOf(ArrowTable);
+    });
+  },
+);
 
 describe("merge insert", () => {
   let tmpDir: tmp.DirResult;
@@ -315,7 +375,7 @@ describe("When creating an index", () => {
       .query()
       .limit(2)
       .nearestTo(queryVec)
-      .distanceType("DoT")
+      .distanceType("dot")
       .toArrow();
     expect(rst.numRows).toBe(2);
 
@@ -648,98 +708,151 @@ describe("when optimizing a dataset", () => {
   });
 });
 
-describe("table.search", () => {
+describe.each([arrow13, arrow14, arrow15, arrow16, arrow17])(
-  let tmpDir: tmp.DirResult;
+  "when optimizing a dataset",
-  beforeEach(() => {
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  (arrow: any) => {
-  });
+    let tmpDir: tmp.DirResult;
-  afterEach(() => tmpDir.removeCallback());
+    beforeEach(() => {
+      getRegistry().reset();
+      tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    });
+    afterEach(() => {
+      tmpDir.removeCallback();
+    });
 
-  test("can search using a string", async () => {
+    test("can search using a string", async () => {
-    @register()
+      @register()
-    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      class MockEmbeddingFunction extends EmbeddingFunction<string> {
-      toJSON(): object {
+        toJSON(): object {
-        return {};
+          return {};
-      }
+        }
-      ndims() {
+        ndims() {
-        return 1;
+          return 1;
-      }
+        }
-      embeddingDataType(): arrow.Float {
+        embeddingDataType() {
-        return new Float32();
+          return new Float32();
-      }
-
-      // Hardcoded embeddings for the sake of testing
-      async computeQueryEmbeddings(_data: string) {
-        switch (_data) {
-          case "greetings":
-            return [0.1];
-          case "farewell":
-            return [0.2];
-          default:
-            return null as never;
         }
-      }
 
-      // Hardcoded embeddings for the sake of testing
+        // Hardcoded embeddings for the sake of testing
-      async computeSourceEmbeddings(data: string[]) {
+        async computeQueryEmbeddings(_data: string) {
-        return data.map((s) => {
+          switch (_data) {
-          switch (s) {
+            case "greetings":
-            case "hello world":
               return [0.1];
-            case "goodbye world":
+            case "farewell":
               return [0.2];
             default:
               return null as never;
           }
-        });
+        }
+
+        // Hardcoded embeddings for the sake of testing
+        async computeSourceEmbeddings(data: string[]) {
+          return data.map((s) => {
+            switch (s) {
+              case "hello world":
+                return [0.1];
+              case "goodbye world":
+                return [0.2];
+              default:
+                return null as never;
+            }
+          });
+        }
       }
-    }
 
-    const func = new MockEmbeddingFunction();
+      const func = new MockEmbeddingFunction();
-    const schema = LanceSchema({
+      const schema = LanceSchema({
-      text: func.sourceField(new arrow.Utf8()),
+        text: func.sourceField(new arrow.Utf8()),
-      vector: func.vectorField(),
+        vector: func.vectorField(),
+      });
+      const db = await connect(tmpDir.name);
+      const data = [{ text: "hello world" }, { text: "goodbye world" }];
+      const table = await db.createTable("test", data, { schema });
+
+      const results = await table.search("greetings").toArray();
+      expect(results[0].text).toBe(data[0].text);
+
+      const results2 = await table.search("farewell").toArray();
+      expect(results2[0].text).toBe(data[1].text);
     });
-    const db = await connect(tmpDir.name);
-    const data = [{ text: "hello world" }, { text: "goodbye world" }];
-    const table = await db.createTable("test", data, { schema });
 
-    const results = await table.search("greetings").then((r) => r.toArray());
+    test("rejects if no embedding function provided", async () => {
-    expect(results[0].text).toBe(data[0].text);
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "hello world", vector: [0.1, 0.2, 0.3] },
+        { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
 
-    const results2 = await table.search("farewell").then((r) => r.toArray());
+      expect(table.search("hello").toArray()).rejects.toThrow(
-    expect(results2[0].text).toBe(data[1].text);
+        "No embedding functions are defined in the table",
+      );
+    });
+
+    test.each([
+      [0.4, 0.5, 0.599], // number[]
+      Float32Array.of(0.4, 0.5, 0.599), // Float32Array
+      Float64Array.of(0.4, 0.5, 0.599), // Float64Array
+    ])("can search using vectorlike datatypes", async (vectorlike) => {
+      const db = await connect(tmpDir.name);
+      const data = [
+        { text: "hello world", vector: [0.1, 0.2, 0.3] },
+        { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
+      ];
+      const table = await db.createTable("test", data);
+
+      // biome-ignore lint/suspicious/noExplicitAny: test
+      const results: any[] = await table.search(vectorlike).toArray();
+
+      expect(results.length).toBe(2);
+      expect(results[0].text).toBe(data[1].text);
+    });
+  },
+);
+
+describe("when calling explainPlan", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+  let queryVec: number[];
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const con = await connect(tmpDir.name);
+    table = await con.createTable("vectors", [{ id: 1, vector: [0.1, 0.2] }]);
   });
 
-  test("rejects if no embedding function provided", async () => {
+  afterEach(() => {
-    const db = await connect(tmpDir.name);
+    tmpDir.removeCallback();
-    const data = [
-      { text: "hello world", vector: [0.1, 0.2, 0.3] },
-      { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
-    ];
-    const table = await db.createTable("test", data);
-
-    expect(table.search("hello")).rejects.toThrow(
-      "No embedding functions are defined in the table",
-    );
   });
 
-  test.each([
+  it("retrieves query plan", async () => {
-    [0.4, 0.5, 0.599], // number[]
+    queryVec = Array(2)
-    Float32Array.of(0.4, 0.5, 0.599), // Float32Array
+      .fill(1)
-    Float64Array.of(0.4, 0.5, 0.599), // Float64Array
+      .map(() => Math.random());
-  ])("can search using vectorlike datatypes", async (vectorlike) => {
+    const plan = await table.query().nearestTo(queryVec).explainPlan(true);
-    const db = await connect(tmpDir.name);
-    const data = [
-      { text: "hello world", vector: [0.1, 0.2, 0.3] },
-      { text: "goodbye world", vector: [0.4, 0.5, 0.6] },
-    ];
-    const table = await db.createTable("test", data);
 
-    // biome-ignore lint/suspicious/noExplicitAny: test
+    expect(plan).toMatch("KNN");
-    const results: any[] = await table.search(vectorlike).toArray();
+  });
-
+});
-    expect(results.length).toBe(2);
+
-    expect(results[0].text).toBe(data[1].text);
+describe("column name options", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const con = await connect(tmpDir.name);
+    table = await con.createTable("vectors", [
+      { camelCase: 1, vector: [0.1, 0.2] },
+    ]);
+  });
+
+  test("can select columns with different names", async () => {
+    const results = await table.query().select(["camelCase"]).toArray();
+    expect(results[0].camelCase).toBe(1);
+  });
+
+  test("can filter on columns with different names", async () => {
+    const results = await table.query().where("`camelCase` = 1").toArray();
+    expect(results[0].camelCase).toBe(1);
   });
 });

nodejs/__test__/tsconfig.json

@@ -6,5 +6,5 @@
     "target": "es2022",
     "types": ["jest", "node"]
   },
-  "include": ["**/*"]
+  "include": ["**/*", "../examples/ann_indexes.ts"]
 }

nodejs/__test__/util.test.ts

@@ -0,0 +1,28 @@
+import { IntoSql, toSQL } from "../lancedb/util";
+test.each([
+  ["string", "'string'"],
+  [123, "123"],
+  [1.11, "1.11"],
+  [true, "TRUE"],
+  [false, "FALSE"],
+  [null, "NULL"],
+  [new Date("2021-01-01T00:00:00.000Z"), "'2021-01-01T00:00:00.000Z'"],
+  [[1, 2, 3], "[1, 2, 3]"],
+  [new ArrayBuffer(8), "X'0000000000000000'"],
+  [Buffer.from("hello"), "X'68656c6c6f'"],
+  ["Hello 'world'", "'Hello ''world'''"],
+])("toSQL(%p) === %p", (value, expected) => {
+  expect(toSQL(value)).toBe(expected);
+});
+
+test("toSQL({}) throws on unsupported value type", () => {
+  expect(() => toSQL({} as unknown as IntoSql)).toThrow(
+    "Unsupported value type: object value: ([object Object])",
+  );
+});
+test("toSQL() throws on unsupported value type", () => {
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  expect(() => (<any>toSQL)()).toThrow(
+    "Unsupported value type: undefined value: (undefined)",
+  );
+});

nodejs/biome.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "https://biomejs.dev/schemas/1.7.3/schema.json",
+  "$schema": "https://biomejs.dev/schemas/1.8.3/schema.json",
   "organizeImports": {
     "enabled": true
   },
@@ -94,12 +94,28 @@
         "useValidTypeof": "error"
       }
     },
-    "ignore": ["**/dist/**/*", "**/native.js", "**/native.d.ts"]
+    "ignore": [
+      "**/dist/**/*",
+      "**/native.js",
+      "**/native.d.ts",
+      "__test__/docs/**/*",
+      "examples/**/*"
+    ]
   },
   "javascript": {
     "globals": []
   },
   "overrides": [
+    {
+      "include": ["__test__/s3_integration.test.ts"],
+      "linter": {
+        "rules": {
+          "style": {
+            "useNamingConvention": "off"
+          }
+        }
+      }
+    },
     {
       "include": [
         "**/*.ts",

nodejs/examples/.gitignore

@@ -0,0 +1 @@
+data/

nodejs/examples/ann_indexes.ts

@@ -0,0 +1,49 @@
+// --8<-- [start:import]
+import * as lancedb from "@lancedb/lancedb";
+// --8<-- [end:import]
+
+// --8<-- [start:ingest]
+const db = await lancedb.connect("/tmp/lancedb/");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: `${i}`,
+  content: "",
+  longId: `${i}`,
+}));
+
+const table = await db.createTable("my_vectors", data, { mode: "overwrite" });
+await table.createIndex("vector", {
+  config: lancedb.Index.ivfPq({
+    numPartitions: 16,
+    numSubVectors: 48,
+  }),
+});
+// --8<-- [end:ingest]
+
+// --8<-- [start:search1]
+const _results1 = await table
+  .search(Array(1536).fill(1.2))
+  .limit(2)
+  .nprobes(20)
+  .refineFactor(10)
+  .toArray();
+// --8<-- [end:search1]
+
+// --8<-- [start:search2]
+const _results2 = await table
+  .search(Array(1536).fill(1.2))
+  .where("id != '1141'")
+  .limit(2)
+  .toArray();
+// --8<-- [end:search2]
+
+// --8<-- [start:search3]
+const _results3 = await table
+  .search(Array(1536).fill(1.2))
+  .select(["id"])
+  .limit(2)
+  .toArray();
+// --8<-- [end:search3]
+
+console.log("Ann indexes: done");

nodejs/examples/basic.ts

@@ -0,0 +1,162 @@
+// --8<--  [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import * as arrow from "apache-arrow";
+import {
+  Field,
+  FixedSizeList,
+  Float16,
+  Int32,
+  Schema,
+  Utf8,
+} from "apache-arrow";
+
+// --8<-- [end:imports]
+
+// --8<-- [start:connect]
+const uri = "/tmp/lancedb/";
+const db = await lancedb.connect(uri);
+// --8<-- [end:connect]
+{
+  // --8<-- [start:create_table]
+  const tbl = await db.createTable(
+    "myTable",
+    [
+      { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+      { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+    ],
+    { mode: "overwrite" },
+  );
+  // --8<-- [end:create_table]
+
+  const data = [
+    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+  ];
+
+  {
+    // --8<-- [start:create_table_exists_ok]
+    const tbl = await db.createTable("myTable", data, {
+      existsOk: true,
+    });
+    // --8<-- [end:create_table_exists_ok]
+  }
+  {
+    // --8<-- [start:create_table_overwrite]
+    const _tbl = await db.createTable("myTable", data, {
+      mode: "overwrite",
+    });
+    // --8<-- [end:create_table_overwrite]
+  }
+}
+
+{
+  // --8<-- [start:create_table_with_schema]
+  const schema = new arrow.Schema([
+    new arrow.Field(
+      "vector",
+      new arrow.FixedSizeList(
+        2,
+        new arrow.Field("item", new arrow.Float32(), true),
+      ),
+    ),
+    new arrow.Field("item", new arrow.Utf8(), true),
+    new arrow.Field("price", new arrow.Float32(), true),
+  ]);
+  const data = [
+    { vector: [3.1, 4.1], item: "foo", price: 10.0 },
+    { vector: [5.9, 26.5], item: "bar", price: 20.0 },
+  ];
+  const _tbl = await db.createTable("myTable", data, {
+    schema,
+  });
+  // --8<-- [end:create_table_with_schema]
+}
+
+{
+  // --8<-- [start:create_empty_table]
+
+  const schema = new arrow.Schema([
+    new arrow.Field("id", new arrow.Int32()),
+    new arrow.Field("name", new arrow.Utf8()),
+  ]);
+
+  const empty_tbl = await db.createEmptyTable("empty_table", schema);
+  // --8<-- [end:create_empty_table]
+}
+{
+  // --8<-- [start:open_table]
+  const _tbl = await db.openTable("myTable");
+  // --8<-- [end:open_table]
+}
+
+{
+  // --8<-- [start:table_names]
+  const tableNames = await db.tableNames();
+  console.log(tableNames);
+  // --8<-- [end:table_names]
+}
+
+const tbl = await db.openTable("myTable");
+{
+  // --8<-- [start:add_data]
+  const data = [
+    { vector: [1.3, 1.4], item: "fizz", price: 100.0 },
+    { vector: [9.5, 56.2], item: "buzz", price: 200.0 },
+  ];
+  await tbl.add(data);
+  // --8<-- [end:add_data]
+}
+{
+  // --8<-- [start:vector_search]
+  const _res = tbl.search([100, 100]).limit(2).toArray();
+  // --8<-- [end:vector_search]
+}
+{
+  const data = Array.from({ length: 1000 })
+    .fill(null)
+    .map(() => ({
+      vector: [Math.random(), Math.random()],
+      item: "autogen",
+      price: Math.round(Math.random() * 100),
+    }));
+
+  await tbl.add(data);
+}
+
+// --8<-- [start:create_index]
+await tbl.createIndex("vector");
+// --8<-- [end:create_index]
+
+// --8<-- [start:delete_rows]
+await tbl.delete('item = "fizz"');
+// --8<-- [end:delete_rows]
+
+// --8<-- [start:drop_table]
+await db.dropTable("myTable");
+// --8<-- [end:drop_table]
+await db.dropTable("empty_table");
+
+{
+  // --8<-- [start:create_f16_table]
+  const db = await lancedb.connect("/tmp/lancedb");
+  const dim = 16;
+  const total = 10;
+  const f16Schema = new Schema([
+    new Field("id", new Int32()),
+    new Field(
+      "vector",
+      new FixedSizeList(dim, new Field("item", new Float16(), true)),
+      false,
+    ),
+  ]);
+  const data = lancedb.makeArrowTable(
+    Array.from(Array(total), (_, i) => ({
+      id: i,
+      vector: Array.from(Array(dim), Math.random),
+    })),
+    { schema: f16Schema },
+  );
+  const _table = await db.createTable("f16_tbl", data);
+  // --8<-- [end:create_f16_table]
+  await db.dropTable("f16_tbl");
+}

nodejs/examples/embedding.ts

@@ -0,0 +1,83 @@
+// --8<-- [start:imports]
+import * as lancedb from "@lancedb/lancedb";
+import { LanceSchema, getRegistry, register } from "@lancedb/lancedb/embedding";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { type Float, Float32, Utf8 } from "apache-arrow";
+// --8<-- [end:imports]
+
+{
+  // --8<-- [start:openai_embeddings]
+
+  const db = await lancedb.connect("/tmp/db");
+  const func = getRegistry()
+    .get("openai")
+    ?.create({ model: "text-embedding-ada-002" }) as EmbeddingFunction;
+
+  const wordsSchema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+  const tbl = await db.createEmptyTable("words", wordsSchema, {
+    mode: "overwrite",
+  });
+  await tbl.add([{ text: "hello world" }, { text: "goodbye world" }]);
+
+  const query = "greetings";
+  const actual = (await (await tbl.search(query)).limit(1).toArray())[0];
+
+  // --8<-- [end:openai_embeddings]
+  console.log("result = ", actual.text);
+}
+
+{
+  // --8<-- [start:embedding_function]
+  const db = await lancedb.connect("/tmp/db");
+
+  @register("my_embedding")
+  class MyEmbeddingFunction extends EmbeddingFunction<string> {
+    toJSON(): object {
+      return {};
+    }
+    ndims() {
+      return 3;
+    }
+    embeddingDataType(): Float {
+      return new Float32();
+    }
+    async computeQueryEmbeddings(_data: string) {
+      // This is a placeholder for a real embedding function
+      return [1, 2, 3];
+    }
+    async computeSourceEmbeddings(data: string[]) {
+      // This is a placeholder for a real embedding function
+      return Array.from({ length: data.length }).fill([1, 2, 3]) as number[][];
+    }
+  }
+
+  const func = new MyEmbeddingFunction();
+
+  const data = [{ text: "pepperoni" }, { text: "pineapple" }];
+
+  // Option 1: manually specify the embedding function
+  const table = await db.createTable("vectors", data, {
+    embeddingFunction: {
+      function: func,
+      sourceColumn: "text",
+      vectorColumn: "vector",
+    },
+    mode: "overwrite",
+  });
+
+  // Option 2: provide the embedding function through a schema
+
+  const schema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+
+  const table2 = await db.createTable("vectors2", data, {
+    schema,
+    mode: "overwrite",
+  });
+  // --8<-- [end:embedding_function]
+}

nodejs/examples/filtering.ts

@@ -0,0 +1,34 @@
+import * as lancedb from "@lancedb/lancedb";
+
+const db = await lancedb.connect("data/sample-lancedb");
+
+const data = Array.from({ length: 10_000 }, (_, i) => ({
+  vector: Array(1536).fill(i),
+  id: i,
+  item: `item ${i}`,
+  strId: `${i}`,
+}));
+
+const tbl = await db.createTable("myVectors", data, { mode: "overwrite" });
+
+// --8<-- [start:search]
+const _result = await tbl
+  .search(Array(1536).fill(0.5))
+  .limit(1)
+  .where("id = 10")
+  .toArray();
+// --8<-- [end:search]
+
+// --8<-- [start:vec_search]
+await tbl
+  .search(Array(1536).fill(0))
+  .where("(item IN ('item 0', 'item 2')) AND (id > 10)")
+  .postfilter()
+  .toArray();
+// --8<-- [end:vec_search]
+
+// --8<-- [start:sql_search]
+await tbl.query().where("id = 10").limit(10).toArray();
+// --8<-- [end:sql_search]
+
+console.log("SQL search: done");

nodejs/examples/jsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

nodejs/examples/package-lock.json

@@ -0,0 +1,79 @@
+{
+  "name": "examples",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "examples",
+      "version": "1.0.0",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@lancedb/lancedb": "file:../"
+      },
+      "peerDependencies": {
+        "typescript": "^5.0.0"
+      }
+    },
+    "..": {
+      "name": "@lancedb/lancedb",
+      "version": "0.6.0",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "apache-arrow": "^15.0.0",
+        "axios": "^1.7.2",
+        "openai": "^4.29.2",
+        "reflect-metadata": "^0.2.2"
+      },
+      "devDependencies": {
+        "@aws-sdk/client-kms": "^3.33.0",
+        "@aws-sdk/client-s3": "^3.33.0",
+        "@biomejs/biome": "^1.7.3",
+        "@jest/globals": "^29.7.0",
+        "@napi-rs/cli": "^2.18.0",
+        "@types/axios": "^0.14.0",
+        "@types/jest": "^29.1.2",
+        "@types/tmp": "^0.2.6",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
+        "eslint": "^8.57.0",
+        "jest": "^29.7.0",
+        "shx": "^0.3.4",
+        "tmp": "^0.2.3",
+        "ts-jest": "^29.1.2",
+        "typedoc": "^0.25.7",
+        "typedoc-plugin-markdown": "^3.17.1",
+        "typescript": "^5.3.3",
+        "typescript-eslint": "^7.1.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      }
+    },
+    "node_modules/@lancedb/lancedb": {
+      "resolved": "..",
+      "link": true
+    },
+    "node_modules/typescript": {
+      "version": "5.5.2",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.2.tgz",
+      "integrity": "sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew==",
+      "peer": true,
+      "bin": {
+        "tsc": "bin/tsc",
+        "tsserver": "bin/tsserver"
+      },
+      "engines": {
+        "node": ">=14.17"
+      }
+    }
+  }
+}

nodejs/examples/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "examples",
+  "version": "1.0.0",
+  "description": "Examples for LanceDB",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@lancedb/lancedb": "file:../"
+  },
+  "peerDependencies": {
+    "typescript": "^5.0.0"
+  }
+}