[python] Bump version: 0.3.3 → 0.3.4

Updating package-lock.json
Bump version: 0.3.7 → 0.3.8
2025-12-23 21:39:57 +00:00 · 2023-11-19 00:24:01 +00:00 · 2023-11-19 00:04:08 +00:00 · 2023-11-19 00:03:58 +00:00 · 2023-11-18 15:57:23 -08:00 · 2023-11-18 14:17:40 -08:00
53 changed files with 1331 additions and 299 deletions
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.3
+current_version = 0.3.8
 commit = True
 message = Bump version: {current_version} → {new_version}
 tag = True
--- a/.github/workflows/node.yml
+++ b/.github/workflows/node.yml
@@ -11,6 +11,10 @@ on:
      - .github/workflows/node.yml
      - docker-compose.yml

+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
  # Disable full debug symbol generation to speed up CI build and keep memory down
  # "1" means line tables only, which is useful for panic tracebacks.
--- a/.github/workflows/npm-publish.yml
+++ b/.github/workflows/npm-publish.yml
@@ -38,7 +38,7 @@ jobs:
            node/vectordb-*.tgz

  node-macos:
-    runs-on: macos-12
+    runs-on: macos-13
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    strategy:
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -8,6 +8,11 @@ on:
    paths:
      - python/**
      - .github/workflows/python.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
  linux:
    timeout-minutes: 30
@@ -32,18 +37,19 @@ jobs:
      run: |
        pip install -e .[tests]
        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
-        pip install pytest pytest-mock black isort
-    - name: Black
-      run: black --check --diff --no-color --quiet .
-    - name: isort
-      run: isort --check --diff --quiet .
+        pip install pytest pytest-mock ruff
+    - name: Lint
+      run: ruff format --check .
    - name: Run tests
      run: pytest -m "not slow" -x -v --durations=30 tests
    - name: doctest
      run: pytest --doctest-modules lancedb
  mac:
    timeout-minutes: 30
-    runs-on: "macos-12"
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
    defaults:
      run:
        shell: bash
@@ -62,8 +68,6 @@ jobs:
        pip install -e .[tests]
        pip install tantivy@git+https://github.com/quickwit-oss/tantivy-py#164adc87e1a033117001cf70e38c82a53014d985
        pip install pytest pytest-mock black
-    - name: Black
-      run: black --check --diff --no-color --quiet .
    - name: Run tests
      run: pytest -m "not slow" -x -v --durations=30 tests
  pydantic1x:
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -10,6 +10,10 @@ on:
      - rust/**
      - .github/workflows/rust.yml

+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
@@ -44,8 +48,11 @@ jobs:
    - name: Run tests
      run: cargo test --all-features
  macos:
-    runs-on: macos-12
    timeout-minutes: 30
+    strategy:
+      matrix:
+        mac-runner: [ "macos-13", "macos-13-xlarge" ]
+    runs-on: "${{ matrix.mac-runner }}"
    defaults:
      run:
        shell: bash
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -5,9 +5,10 @@ exclude = ["python"]
 resolver = "2"

 [workspace.dependencies]
-lance = { "version" = "=0.8.7", "features" = ["dynamodb"] }
-lance-linalg = { "version" = "=0.8.7" }
-lance-testing = { "version" = "=0.8.7" }
+lance = { "version" = "=0.8.17", "features" = ["dynamodb"] }
+lance-index = { "version" = "=0.8.17" }
+lance-linalg = { "version" = "=0.8.17" }
+lance-testing = { "version" = "=0.8.17" }
 # Note that this one does not include pyarrow
 arrow = { version = "47.0.0", optional = false }
 arrow-array = "47.0"
@@ -19,7 +20,7 @@ arrow-arith = "47.0"
 arrow-cast = "47.0"
 chrono = "0.4.23"
 half = { "version" = "=2.3.1", default-features = false, features = [
-    "num-traits"
+    "num-traits",
 ] }
 log = "0.4"
 object_store = "0.7.1"
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -1,4 +1,5 @@
 site_name: LanceDB Docs
+site_url: https://lancedb.github.io/lancedb/
 repo_url: https://github.com/lancedb/lancedb
 edit_uri: https://github.com/lancedb/lancedb/tree/main/docs/src
 repo_name: lancedb/lancedb
@@ -150,8 +151,6 @@ nav:

 extra_css:
  - styles/global.css
-extra_javascript:
-  - scripts/posthog.js

 extra:
  analytics:
--- a/docs/src/ann_indexes.md
+++ b/docs/src/ann_indexes.md
@@ -71,9 +71,41 @@ a single PQ code.
 ### Use GPU to build vector index

 Lance Python SDK has experimental GPU support for creating IVF index.
+Using GPU for index creation requires [PyTorch>2.0](https://pytorch.org/) being installed.
+
 You can specify the GPU device to train IVF partitions via

- **accelerator**: Specify to `"cuda"`` to enable GPU training.
+- **accelerator**: Specify to ``cuda`` or ``mps`` (on Apple Silicon) to enable GPU training.
+
+=== "Linux"
+
+     <!-- skip-test -->
+     ``` { .python .copy }
+     # Create index using CUDA on Nvidia GPUs.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="cuda"
+     )
+     ```
+
+=== "Macos"
+
+     <!-- skip-test -->
+     ```python
+     # Create index using MPS on Apple Silicon.
+     tbl.create_index(
+          num_partitions=256,
+          num_sub_vectors=96,
+          accelerator="mps"
+     )
+     ```
+
+Trouble shootings:
+
+If you see ``AssertionError: Torch not compiled with CUDA enabled``, you need to [install
+PyTorch with CUDA support](https://pytorch.org/get-started/locally/).
+

 ## Querying an ANN Index

--- a/docs/src/embeddings/default_embedding_functions.md
+++ b/docs/src/embeddings/default_embedding_functions.md
@@ -1,7 +1,9 @@
 There are various Embedding functions available out of the box with lancedb. We're working on supporting other popular embedding APIs.

 ## Text Embedding Functions
-Here are the text embedding functions registered by default
+Here are the text embedding functions registered by default.
+Embedding functions have inbuilt rate limit handler wrapper for source and query embedding function calls that retry with exponential standoff. 
+Each `EmbeddingFunction` implementation automatically takes `max_retries` as an argument which has the deafult value of 7.

 ### Sentence Transformers
 Here are the parameters that you can set when registering a `sentence-transformers` object, and their default values:
@@ -66,6 +68,56 @@ actual = table.search(query).limit(1).to_pydantic(Words)[0]
 print(actual.text)
 ```

+### Instructor Embeddings
+Instructor is an instruction-finetuned text embedding model that can generate text embeddings tailored to any task (e.g., classification, retrieval, clustering, text evaluation, etc.) and domains (e.g., science, finance, etc.) by simply providing the task instruction, without any finetuning
+
+If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+
+                          Represent the `domain` `text_type` for `task_objective`:
+
+* `domain` is optional, and it specifies the domain of the text, e.g., science, finance, medicine, etc.
+* `text_type` is required, and it specifies the encoding unit, e.g., sentence, document, paragraph, etc.
+* `task_objective` is optional, and it specifies the objective of embedding, e.g., retrieve a document, classify the sentence, etc.
+
+More information about the model can be found here - https://github.com/xlang-ai/instructor-embedding
+
+| Argument | Type | Default | Description |
+|---|---|---|---|
+| `name` | `str` | "hkunlp/instructor-base" | The name of the model to use |
+| `batch_size` | `int` | `32` | The batch size to use when generating embeddings |
+| `device` | `str` | `"cpu"` | The device to use when generating embeddings |
+| `show_progress_bar` | `bool` | `True` | Whether to show a progress bar when generating embeddings |
+| `normalize_embeddings` | `bool` | `True` | Whether to normalize the embeddings |
+| `quantize` | `bool` | `False` | Whether to quantize the model |
+| `source_instruction` | `str` | `"represent the docuement for retreival"` | The instruction for the source column |
+| `query_instruction` | `str` | `"represent the document for retreiving the most similar documents"` | The instruction for the query |
+
+
+
+```python
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+instructor = get_registry().get("instructor").create(
+                            source_instruction="represent the docuement for retreival",
+                            query_instruction="represent the document for retreiving the most similar documents"
+                            )
+
+class Schema(LanceModel):
+    vector: Vector(instructor.ndims()) = instructor.VectorField()
+    text: str = instructor.SourceField()
+
+db = lancedb.connect("~/.lancedb")
+tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+        {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+        {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+tbl.add(texts)
+```
+
 ## Multi-modal embedding functions
 Multi-modal embedding functions allow you query your table using both images and text.

--- a/docs/src/embeddings/embedding_functions.md
+++ b/docs/src/embeddings/embedding_functions.md
@@ -57,6 +57,19 @@ query_image = Image.open(p)
 table.search(query_image)

 ```
+### Rate limit Handling
+`EmbeddingFunction` class wraps the calls for source and query embedding generation inside a rate limit handler that retries the requests with exponential backoff after successive failures. By default the maximum retires is set to 7. You can tune it by setting it to a different number or disable it by setting it to 0.
+Example
+----
+
+```python
+clip = registry.get("open-clip").create() # Defaults to 7 max retries
+clip = registry.get("open-clip").create(max_retries=10) # Increase max retries to 10
+clip = registry.get("open-clip").create(max_retries=0) # Retries disabled
+````
+
+NOTE:
+Embedding functions can also fail due to other errors that have nothing to do with rate limits. This is why the error is also logged.

 ### A little fun with PyDantic
 LanceDB is integrated with PyDantic. Infact we've used the integration in the above example to define the schema. It is also being used behing the scene by the embdding function API to ingest useful information as table metadata.
--- a/docs/src/python/python.md
+++ b/docs/src/python/python.md
@@ -22,8 +22,6 @@ pip install lancedb

 ::: lancedb.query.LanceQueryBuilder

-::: lancedb.query.LanceFtsQueryBuilder
-
 ## Embeddings

 ::: lancedb.embeddings.registry.EmbeddingFunctionRegistry
@@ -56,7 +54,7 @@ pip install lancedb

 ## Utilities

-::: lancedb.vector
+::: lancedb.schema.vector

 ## Integrations

--- a/docs/src/robots.txt
+++ b/docs/src/robots.txt
@@ -0,0 +1 @@
+User-agent: *
--- a/docs/test/md_testing.py
+++ b/docs/test/md_testing.py
@@ -18,20 +18,31 @@ python_file = ".py"
 python_folder = "python"

 files = glob.glob(glob_string, recursive=True)
-excluded_files = [f for excluded_glob in excluded_globs for f in glob.glob(excluded_glob, recursive=True)]
+excluded_files = [
+    f
+    for excluded_glob in excluded_globs
+    for f in glob.glob(excluded_glob, recursive=True)
+]
+

 def yield_lines(lines: Iterator[str], prefix: str, suffix: str):
    in_code_block = False
    # Python code has strict indentation
    strip_length = 0
+    skip_test = False
    for line in lines:
+        if "skip-test" in line:
+            skip_test = True
        if line.strip().startswith(prefix + python_prefix):
            in_code_block = True
            strip_length = len(line) - len(line.lstrip())
        elif in_code_block and line.strip().startswith(suffix):
            in_code_block = False
+            if not skip_test:
                yield "\n"
+            skip_test = False
        elif in_code_block:
+            if not skip_test:
                yield line[strip_length:]

 for file in filter(lambda file: file not in excluded_files, files):
@@ -39,7 +50,12 @@ for file in filter(lambda file: file not in excluded_files, files):
        lines = list(yield_lines(iter(f), "```", "```"))

    if len(lines) > 0:
-        out_path = Path(python_folder) / Path(file).name.strip(".md") / (Path(file).name.strip(".md") + python_file)
+        print(lines)
+        out_path = (
+            Path(python_folder)
+            / Path(file).name.strip(".md")
+            / (Path(file).name.strip(".md") + python_file)
+        )
        print(out_path)
        out_path.parent.mkdir(exist_ok=True, parents=True)
        with open(out_path, "w") as out:
--- a/node/README.md
+++ b/node/README.md
@@ -10,7 +10,7 @@ npm install vectordb

 This will download the appropriate native library for your platform. We currently
 support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support Windows or musl-based Linux (such as Alpine Linux).
+yet support musl-based Linux (such as Alpine Linux).

 ## Usage

--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "vectordb",
-  "version": "0.3.3",
+  "version": "0.3.8",
  "lockfileVersion": 2,
  "requires": true,
  "packages": {
    "": {
      "name": "vectordb",
-      "version": "0.3.3",
+      "version": "0.3.8",
      "cpu": [
        "x64",
        "arm64"
@@ -53,11 +53,11 @@
        "uuid": "^9.0.0"
      },
      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.3.3",
-        "@lancedb/vectordb-darwin-x64": "0.3.3",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.3.3",
-        "@lancedb/vectordb-linux-x64-gnu": "0.3.3",
-        "@lancedb/vectordb-win32-x64-msvc": "0.3.3"
+        "@lancedb/vectordb-darwin-arm64": "0.3.8",
+        "@lancedb/vectordb-darwin-x64": "0.3.8",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.3.8",
+        "@lancedb/vectordb-linux-x64-gnu": "0.3.8",
+        "@lancedb/vectordb-win32-x64-msvc": "0.3.8"
      }
    },
    "node_modules/@apache-arrow/ts": {
@@ -316,66 +316,6 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
-    "node_modules/@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.3.tgz",
-      "integrity": "sha512-nvyj7xNX2/wb/PH5TjyhLR/NQ1jVuoBw2B5UaSg7qf8Tnm5SSXWQ7F25RVKcKwh72fz1qB+CWW24ftZnRzbT/Q==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.3.tgz",
-      "integrity": "sha512-7CW+nILyPHp6cua0Rl0xaTDWw/vajEn/jCsEjFYgDmE+rtf5Z5Fum41FxR9C2TtIAvUK+nWb5mkYeOLqU6vRvg==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "darwin"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.3.tgz",
-      "integrity": "sha512-MmhwbacKxZPkLwwOqysVY8mUb8lFoyFIPlYhSLV4xS1C8X4HWALljIul1qMl1RYudp9Uc3PsOzRexl+OvCGfUw==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.3.tgz",
-      "integrity": "sha512-OrNlsKi/QPw59Po040oRKn8IuqFEk4upc/4FaFKqVkcmQjjZrMg5Kgy9ZfWIhHdAnWXXggZZIPArpt0X1B0ceA==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux"
-      ]
-    },
-    "node_modules/@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.3.tgz",
-      "integrity": "sha512-lIT0A7a6eqX51IfGyhECtpXXgsr//kgbd+HZbcCdPy2GMmNezSch/7V22zExDSpF32hX8WfgcTLYCVWVilggDQ==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ]
-    },
    "node_modules/@neon-rs/cli": {
      "version": "0.0.160",
      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
@@ -4868,36 +4808,6 @@
        "@jridgewell/sourcemap-codec": "^1.4.10"
      }
    },
-    "@lancedb/vectordb-darwin-arm64": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.3.3.tgz",
-      "integrity": "sha512-nvyj7xNX2/wb/PH5TjyhLR/NQ1jVuoBw2B5UaSg7qf8Tnm5SSXWQ7F25RVKcKwh72fz1qB+CWW24ftZnRzbT/Q==",
-      "optional": true
-    },
-    "@lancedb/vectordb-darwin-x64": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.3.3.tgz",
-      "integrity": "sha512-7CW+nILyPHp6cua0Rl0xaTDWw/vajEn/jCsEjFYgDmE+rtf5Z5Fum41FxR9C2TtIAvUK+nWb5mkYeOLqU6vRvg==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-arm64-gnu": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.3.3.tgz",
-      "integrity": "sha512-MmhwbacKxZPkLwwOqysVY8mUb8lFoyFIPlYhSLV4xS1C8X4HWALljIul1qMl1RYudp9Uc3PsOzRexl+OvCGfUw==",
-      "optional": true
-    },
-    "@lancedb/vectordb-linux-x64-gnu": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.3.3.tgz",
-      "integrity": "sha512-OrNlsKi/QPw59Po040oRKn8IuqFEk4upc/4FaFKqVkcmQjjZrMg5Kgy9ZfWIhHdAnWXXggZZIPArpt0X1B0ceA==",
-      "optional": true
-    },
-    "@lancedb/vectordb-win32-x64-msvc": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.3.3.tgz",
-      "integrity": "sha512-lIT0A7a6eqX51IfGyhECtpXXgsr//kgbd+HZbcCdPy2GMmNezSch/7V22zExDSpF32hX8WfgcTLYCVWVilggDQ==",
-      "optional": true
-    },
    "@neon-rs/cli": {
      "version": "0.0.160",
      "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",
--- a/node/package.json
+++ b/node/package.json
@@ -1,6 +1,6 @@
 {
  "name": "vectordb",
-  "version": "0.3.3",
+  "version": "0.3.8",
  "description": " Serverless, low-latency vector database for AI applications",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
@@ -81,10 +81,10 @@
    }
  },
  "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.3.3",
-    "@lancedb/vectordb-darwin-x64": "0.3.3",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.3.3",
-    "@lancedb/vectordb-linux-x64-gnu": "0.3.3",
-    "@lancedb/vectordb-win32-x64-msvc": "0.3.3"
+    "@lancedb/vectordb-darwin-arm64": "0.3.8",
+    "@lancedb/vectordb-darwin-x64": "0.3.8",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.3.8",
+    "@lancedb/vectordb-linux-x64-gnu": "0.3.8",
+    "@lancedb/vectordb-win32-x64-msvc": "0.3.8"
  }
 }
--- a/node/src/index.ts
+++ b/node/src/index.ts
@@ -23,7 +23,7 @@ import { Query } from './query'
 import { isEmbeddingFunction } from './embedding/embedding_function'

 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableCleanupOldVersions, tableCompactFiles } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')

 export { Query }
 export type { EmbeddingFunction }
@@ -260,6 +260,27 @@ export interface Table<T = number[]> {
   * ```
   */
  delete: (filter: string) => Promise<void>
+
+  /**
+   * List the indicies on this table.
+   */
+  listIndices: () => Promise<VectorIndex[]>
+
+  /**
+   * Get statistics about an index.
+   */
+  indexStats: (indexUuid: string) => Promise<IndexStats>
+}
+
+export interface VectorIndex {
+  columns: string[]
+  name: string
+  uuid: string
+}
+
+export interface IndexStats {
+  numIndexedRows: number | null
+  numUnindexedRows: number | null
 }

 /**
@@ -502,6 +523,14 @@ export class LocalTable<T = number[]> implements Table<T> {
        return res.metrics
      })
  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    return tableListIndices.call(this._tbl)
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    return tableIndexStats.call(this._tbl, indexUuid)
+  }
 }

 export interface CleanupStats {
--- a/node/src/remote/client.ts
+++ b/node/src/remote/client.ts
@@ -63,6 +63,9 @@ export class HttpLancedbClient {
              }
    ).catch((err) => {
      console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
      return err.response
    })
    if (response.status !== 200) {
@@ -86,13 +89,17 @@ export class HttpLancedbClient {
      {
        headers: {
          'Content-Type': 'application/json',
-          'x-api-key': this._apiKey()
+          'x-api-key': this._apiKey(),
+          ...(this._dbName !== undefined ? { 'x-lancedb-database': this._dbName } : {})
        },
        params,
        timeout: 10000
      }
    ).catch((err) => {
      console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
      return err.response
    })
    if (response.status !== 200) {
@@ -128,6 +135,9 @@ export class HttpLancedbClient {
        }
    ).catch((err) => {
      console.error('error: ', err)
+      if (err.response === undefined) {
+        throw new Error(`Network Error: ${err.message as string}`)
+      }
      return err.response
    })
    if (response.status !== 200) {
--- a/node/src/remote/index.ts
+++ b/node/src/remote/index.ts
@@ -14,7 +14,9 @@

 import {
  type EmbeddingFunction, type Table, type VectorIndexParams, type Connection,
-  type ConnectionOptions, type CreateTableOptions, type WriteOptions
+  type ConnectionOptions, type CreateTableOptions, type VectorIndex,
+  type WriteOptions,
+  type IndexStats
 } from '../index'
 import { Query } from '../query'

@@ -235,10 +237,28 @@ export class RemoteTable<T = number[]> implements Table<T> {
  }

  async countRows (): Promise<number> {
-    throw new Error('Not implemented')
+    const result = await this._client.post(`/v1/table/${this._name}/describe/`)
+    return result.data?.stats?.num_rows
  }

  async delete (filter: string): Promise<void> {
    await this._client.post(`/v1/table/${this._name}/delete/`, { predicate: filter })
  }
+
+  async listIndices (): Promise<VectorIndex[]> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/list/`)
+    return results.data.indexes?.map((index: any) => ({
+      columns: index.columns,
+      name: index.index_name,
+      uuid: index.index_uuid
+    }))
+  }
+
+  async indexStats (indexUuid: string): Promise<IndexStats> {
+    const results = await this._client.post(`/v1/table/${this._name}/index/${indexUuid}/stats/`)
+    return {
+      numIndexedRows: results.data.num_indexed_rows,
+      numUnindexedRows: results.data.num_unindexed_rows
+    }
+  }
 }
--- a/node/src/test/test.ts
+++ b/node/src/test/test.ts
@@ -282,7 +282,8 @@ describe('LanceDB client', function () {
      )
      const table = await con.createTable({ name: 'vectors', schema })
      await table.add([{ vector: Array(128).fill(0.1) }])
-      await table.delete('vector IS NOT NULL')
+      // https://github.com/lancedb/lance/issues/1635
+      await table.delete('true')
      const result = await table.search(Array(128).fill(0.1)).execute()
      assert.isEmpty(result)
    })
@@ -328,6 +329,24 @@ describe('LanceDB client', function () {
      const createIndex = table.createIndex({ type: 'ivf_pq', column: 'name', num_partitions: -1, max_iters: 2, num_sub_vectors: 2 })
      await expect(createIndex).to.be.rejectedWith('num_partitions: must be > 0')
    })
+
+    it('should be able to list index and stats', async function () {
+      const uri = await createTestDB(32, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createIndex({ type: 'ivf_pq', column: 'vector', num_partitions: 2, max_iters: 2, num_sub_vectors: 2 })
+
+      const indices = await table.listIndices()
+      expect(indices).to.have.lengthOf(1)
+      expect(indices[0].name).to.equal('vector_idx')
+      expect(indices[0].uuid).to.not.be.equal(undefined)
+      expect(indices[0].columns).to.have.lengthOf(1)
+      expect(indices[0].columns[0]).to.equal('vector')
+
+      const stats = await table.indexStats(indices[0].uuid)
+      expect(stats.numIndexedRows).to.equal(300)
+      expect(stats.numUnindexedRows).to.equal(0)
+    }).timeout(50_000)
  })

  describe('when using a custom embedding function', function () {
@@ -378,6 +397,40 @@ describe('LanceDB client', function () {
  })
 })

+describe('Remote LanceDB client', function () {
+  describe('when the server is not reachable', function () {
+    it('produces a network error', async function () {
+      const con = await lancedb.connect({
+        uri: 'db://test-1234',
+        region: 'asdfasfasfdf',
+        apiKey: 'some-api-key'
+      })
+
+      // GET
+      try {
+        await con.tableNames()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // POST
+      try {
+        await con.createTable({ name: 'vectors', schema: new Schema([]) })
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+
+      // Search
+      const table = await con.openTable('vectors')
+      try {
+        await table.search([0.1, 0.3]).execute()
+      } catch (err) {
+        expect(err).to.have.property('message', 'Network Error: getaddrinfo ENOTFOUND test-1234.asdfasfasfdf.api.lancedb.com')
+      }
+    })
+  })
+})
+
 describe('Query object', function () {
  it('sets custom parameters', async function () {
    const query = new Query([0.1, 0.3])
--- a/python/.bumpversion.cfg
+++ b/python/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.2
+current_version = 0.3.4
 commit = True
 message = [python] Bump version: {current_version} → {new_version}
 tag = True
--- a/python/lancedb/init.py
+++ b/python/lancedb/init.py
@@ -16,10 +16,11 @@ from typing import Optional

 __version__ = importlib.metadata.version("lancedb")

-from .db import URI, DBConnection, LanceDBConnection
+from .common import URI
+from .db import DBConnection, LanceDBConnection
 from .remote.db import RemoteDBConnection
-from .schema import vector
-from .utils import sentry_log
+from .schema import vector  # noqa: F401
+from .utils import sentry_log  # noqa: F401


 def connect(
--- a/python/lancedb/conftest.py
+++ b/python/lancedb/conftest.py
@@ -1,4 +1,6 @@
 import os
+import time
+from typing import Any

 import numpy as np
 import pytest
@@ -38,3 +40,26 @@ class MockTextEmbeddingFunction(TextEmbeddingFunction):

    def ndims(self):
        return 10
+
+
+class RateLimitedAPI:
+    rate_limit = 0.1  # 1 request per 0.1 second
+    last_request_time = 0
+
+    @staticmethod
+    def make_request():
+        current_time = time.time()
+
+        if current_time - RateLimitedAPI.last_request_time < RateLimitedAPI.rate_limit:
+            raise Exception("Rate limit exceeded. Please try again later.")
+
+        # Simulate a successful request
+        RateLimitedAPI.last_request_time = current_time
+        return "Request successful"
+
+
+@registry.register("test-rate-limited")
+class MockRateLimitedEmbeddingFunction(MockTextEmbeddingFunction):
+    def generate_embeddings(self, texts):
+        RateLimitedAPI.make_request()
+        return [self._compute_one_embedding(row) for row in texts]
--- a/python/lancedb/context.py
+++ b/python/lancedb/context.py
@@ -84,7 +84,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
    context windows that don't cross document boundaries. In this case, we can
    pass ``document_id`` as the group by.

-    >>> contextualize(data).window(4).stride(2).text_col('token').groupby('document_id').to_pandas()
+    >>> (contextualize(data)
+    ...     .window(4).stride(2).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                       token  document_id
    0    The quick brown fox            1
    2  brown fox jumped over            1
@@ -92,18 +94,24 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:
    6           the lazy dog            1
    9      I love sandwiches            2

-    ``min_window_size`` determines the minimum size of the  context windows that are generated
-    This can be used to trim the last few context windows which have size less than
-    ``min_window_size``. By default context windows of size 1 are skipped.
+    ``min_window_size`` determines the minimum size of the context windows
+    that are generated.This can be used to trim the last few context windows
+    which have size less than ``min_window_size``.
+    By default context windows of size 1 are skipped.

-    >>> contextualize(data).window(6).stride(3).text_col('token').groupby('document_id').to_pandas()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).text_col('token').groupby('document_id')
+    ...     .to_pandas())
                                 token  document_id
    0  The quick brown fox jumped over            1
    3     fox jumped over the lazy dog            1
    6                     the lazy dog            1
    9                I love sandwiches            2

-    >>> contextualize(data).window(6).stride(3).min_window_size(4).text_col('token').groupby('document_id').to_pandas()
+    >>> (contextualize(data)
+    ...     .window(6).stride(3).min_window_size(4).text_col('token')
+    ...     .groupby('document_id')
+    ...     .to_pandas())
                                 token  document_id
    0  The quick brown fox jumped over            1
    3     fox jumped over the lazy dog            1
@@ -113,7 +121,9 @@ def contextualize(raw_df: "pd.DataFrame") -> Contextualizer:


 class Contextualizer:
-    """Create context windows from a DataFrame. See [lancedb.context.contextualize][]."""
+    """Create context windows from a DataFrame.
+    See [lancedb.context.contextualize][].
+    """

    def __init__(self, raw_df):
        self._text_col = None
@@ -183,7 +193,7 @@ class Contextualizer:
        deprecated_in="0.3.1",
        removed_in="0.4.0",
        current_version=__version__,
-        details="Use the bar function instead",
+        details="Use to_pandas() instead",
    )
    def to_df(self) -> "pd.DataFrame":
        return self.to_pandas()
--- a/python/lancedb/db.py
+++ b/python/lancedb/db.py
@@ -14,26 +14,39 @@
 from __future__ import annotations

 import os
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, Iterable, List, Optional, Union

 import pyarrow as pa
+from overrides import EnforceOverrides, override
 from pyarrow import fs

-from .common import DATA, URI
-from .embeddings import EmbeddingFunctionConfig
-from .pydantic import LanceModel
 from .table import LanceTable, Table
 from .util import fs_from_uri, get_uri_location, get_uri_scheme

+if TYPE_CHECKING:
+    from .common import DATA, URI
+    from .embeddings import EmbeddingFunctionConfig
+    from .pydantic import LanceModel

-class DBConnection(ABC):
+
+class DBConnection(EnforceOverrides):
    """An active LanceDB connection interface."""

    @abstractmethod
-    def table_names(self) -> list[str]:
-        """List all table names in the database."""
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """List all table in this database
+
+        Parameters
+        ----------
+        page_token: str, optional
+            The token to use for pagination. If not present, start from the beginning.
+        limit: int, default 10
+            The size of the page to return.
+        """
        pass

    @abstractmethod
@@ -45,6 +58,7 @@ class DBConnection(ABC):
        mode: str = "create",
        on_bad_vectors: str = "error",
        fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
    ) -> Table:
        """Create a [Table][lancedb.table.Table] in the database.

@@ -52,12 +66,24 @@ class DBConnection(ABC):
        ----------
        name: str
            The name of the table.
-        data: list, tuple, dict, pd.DataFrame; optional
-            The data to initialize the table. User must provide at least one of `data` or `schema`.
-        schema: pyarrow.Schema or LanceModel; optional
-            The schema of the table.
+        data: The data to initialize the table, *optional*
+            User must provide at least one of `data` or `schema`.
+            Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
+        schema: The schema of the table, *optional*
+            Acceptable types are:
+
+            - pyarrow.Schema
+
+            - [LanceModel][lancedb.pydantic.LanceModel]
        mode: str; default "create"
-            The mode to use when creating the table. Can be either "create" or "overwrite".
+            The mode to use when creating the table.
+            Can be either "create" or "overwrite".
            By default, if the table already exists, an exception is raised.
            If you want to overwrite the table, use mode="overwrite".
        on_bad_vectors: str, default "error"
@@ -150,7 +176,8 @@ class DBConnection(ABC):
        ...     for i in range(5):
        ...         yield pa.RecordBatch.from_arrays(
        ...             [
-        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]], pa.list_(pa.float32(), 2)),
+        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],
+        ...                     pa.list_(pa.float32(), 2)),
        ...                 pa.array(["foo", "bar"]),
        ...                 pa.array([10.0, 20.0]),
        ...             ],
@@ -249,12 +276,15 @@ class LanceDBConnection(DBConnection):
    def uri(self) -> str:
        return self._uri

-    def table_names(self) -> list[str]:
-        """Get the names of all tables in the database.
+    @override
+    def table_names(
+        self, page_token: Optional[str] = None, limit: int = 10
+    ) -> Iterable[str]:
+        """Get the names of all tables in the database. The names are sorted.

        Returns
        -------
-        list of str
+        Iterator of str.
            A list of table names.
        """
        try:
@@ -274,6 +304,7 @@ class LanceDBConnection(DBConnection):
            for file_info in paths
            if file_info.extension == "lance"
        ]
+        tables.sort()
        return tables

    def __len__(self) -> int:
@@ -282,6 +313,7 @@ class LanceDBConnection(DBConnection):
    def __contains__(self, name: str) -> bool:
        return name in self.table_names()

+    @override
    def create_table(
        self,
        name: str,
@@ -313,6 +345,7 @@ class LanceDBConnection(DBConnection):
        )
        return tbl

+    @override
    def open_table(self, name: str) -> LanceTable:
        """Open a table in the database.

@@ -327,6 +360,7 @@ class LanceDBConnection(DBConnection):
        """
        return LanceTable.open(self, name)

+    @override
    def drop_table(self, name: str, ignore_missing: bool = False):
        """Drop a table from the database.

@@ -345,6 +379,7 @@ class LanceDBConnection(DBConnection):
            if not ignore_missing:
                raise

+    @override
    def drop_database(self):
        filesystem, path = fs_from_uri(self.uri)
        filesystem.delete_dir(path)
--- a/python/lancedb/embeddings/init.py
+++ b/python/lancedb/embeddings/init.py
@@ -11,8 +11,10 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

+# ruff: noqa: F401
 from .base import EmbeddingFunction, EmbeddingFunctionConfig, TextEmbeddingFunction
 from .cohere import CohereEmbeddingFunction
+from .instructor import InstructorEmbeddingFunction
 from .open_clip import OpenClipEmbeddings
 from .openai import OpenAIEmbeddings
 from .registry import EmbeddingFunctionRegistry, get_registry
--- a/python/lancedb/embeddings/base.py
+++ b/python/lancedb/embeddings/base.py
@@ -1,3 +1,15 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 import importlib
 from abc import ABC, abstractmethod
 from typing import List, Union
@@ -6,7 +18,7 @@ import numpy as np
 import pyarrow as pa
 from pydantic import BaseModel, Field, PrivateAttr

-from .utils import TEXT
+from .utils import TEXT, retry_with_exponential_backoff


 class EmbeddingFunction(BaseModel, ABC):
@@ -21,6 +33,10 @@ class EmbeddingFunction(BaseModel, ABC):
    3. ndims method which returns the number of dimensions of the vector column
    """

+    __slots__ = ("__weakref__",)  # pydantic 1.x compatibility
+    max_retries: int = (
+        7  # Setitng 0 disables retires. Maybe this should not be enabled by default,
+    )
    _ndims: int = PrivateAttr()

    @classmethod
@@ -44,6 +60,25 @@ class EmbeddingFunction(BaseModel, ABC):
        """
        pass

+    def compute_query_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for a given user query with retries
+        """
+        return retry_with_exponential_backoff(
+            self.compute_query_embeddings, max_retries=self.max_retries
+        )(
+            *args,
+            **kwargs,
+        )
+
+    def compute_source_embeddings_with_retry(self, *args, **kwargs) -> List[np.array]:
+        """
+        Compute the embeddings for the source column in the database with retries
+        """
+        return retry_with_exponential_backoff(
+            self.compute_source_embeddings, max_retries=self.max_retries
+        )(*args, **kwargs)
+
    def sanitize_input(self, texts: TEXT) -> Union[List[str], np.ndarray]:
        """
        Sanitize the input to the embedding function.
@@ -103,6 +138,14 @@ class EmbeddingFunction(BaseModel, ABC):
        """
        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)

+    def __eq__(self, __value: object) -> bool:
+        if not hasattr(__value, "__dict__"):
+            return False
+        return vars(self) == vars(__value)
+
+    def __hash__(self) -> int:
+        return hash(frozenset(vars(self).items()))
+

 class EmbeddingFunctionConfig(BaseModel):
    """
--- a/python/lancedb/embeddings/cohere.py
+++ b/python/lancedb/embeddings/cohere.py
@@ -31,7 +31,8 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
    Parameters
    ----------
    name: str, default "embed-multilingual-v2.0"
-        The name of the model to use. See the Cohere documentation for a list of available models.
+        The name of the model to use. See the Cohere documentation for
+        a list of available models.

    Examples
    --------
@@ -39,7 +40,10 @@ class CohereEmbeddingFunction(TextEmbeddingFunction):
    from lancedb.pydantic import LanceModel, Vector
    from lancedb.embeddings import EmbeddingFunctionRegistry

-    cohere = EmbeddingFunctionRegistry.get_instance().get("cohere").create(name="embed-multilingual-v2.0")
+    cohere = EmbeddingFunctionRegistry
+        .get_instance()
+        .get("cohere")
+        .create(name="embed-multilingual-v2.0")

    class TextModel(LanceModel):
        text: str = cohere.SourceField()
--- a/python/lancedb/embeddings/instructor.py
+++ b/python/lancedb/embeddings/instructor.py
@@ -0,0 +1,137 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import List
+
+import numpy as np
+
+from .base import TextEmbeddingFunction
+from .registry import register
+from .utils import TEXT, weak_lru
+
+
+@register("instructor")
+class InstructorEmbeddingFunction(TextEmbeddingFunction):
+    """
+    An embedding function that uses the InstructorEmbedding library. Instructor models support multi-task learning, and can be used for a
+    variety of tasks, including text classification, sentence similarity, and document retrieval.
+    If you want to calculate customized embeddings for specific sentences, you may follow the unified template to write instructions:
+        "Represent the `domain` `text_type` for `task_objective`":
+
+        * domain is optional, and it specifies the domain of the text, e.g., science, finance, medicine, etc.
+        * text_type is required, and it specifies the encoding unit, e.g., sentence, document, paragraph, etc.
+        * task_objective is optional, and it specifies the objective of embedding, e.g., retrieve a document, classify the sentence, etc.
+
+    For example, if you want to calculate embeddings for a document, you may write the instruction as follows:
+        "Represent the document for retreival"
+
+    Parameters
+    ----------
+    name: str
+        The name of the model to use. Available models are listed at https://github.com/xlang-ai/instructor-embedding#model-list;
+        The default model is hkunlp/instructor-base
+    batch_size: int, default 32
+        The batch size to use when generating embeddings
+    device: str, default "cpu"
+        The device to use when generating embeddings
+    show_progress_bar: bool, default True
+        Whether to show a progress bar when generating embeddings
+    normalize_embeddings: bool, default True
+        Whether to normalize the embeddings
+    quantize: bool, default False
+        Whether to quantize the model
+    source_instruction: str, default "represent the docuement for retreival"
+        The instruction for the source column
+    query_instruction: str, default "represent the document for retreiving the most similar documents"
+        The instruction for the query
+
+    Examples
+    --------
+    import lancedb
+    from lancedb.pydantic import LanceModel, Vector
+    from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
+
+    instructor = get_registry().get("instructor").create(
+                                source_instruction="represent the docuement for retreival",
+                                query_instruction="represent the document for retreiving the most similar documents"
+                                )
+
+    class Schema(LanceModel):
+        vector: Vector(instructor.ndims()) = instructor.VectorField()
+        text: str = instructor.SourceField()
+
+    db = lancedb.connect("~/.lancedb")
+    tbl = db.create_table("test", schema=Schema, mode="overwrite")
+
+    texts = [{"text": "Capitalism has been dominant in the Western world since the end of feudalism, but most feel[who?] that..."},
+            {"text": "The disparate impact theory is especially controversial under the Fair Housing Act because the Act..."},
+            {"text": "Disparate impact in United States labor law refers to practices in employment, housing, and other areas that.."}]
+
+    tbl.add(texts)
+
+    """
+
+    name: str = "hkunlp/instructor-base"
+    batch_size: int = 32
+    device: str = "cpu"
+    show_progress_bar: bool = True
+    normalize_embeddings: bool = True
+    quantize: bool = False
+    # convert_to_numpy: bool = True # Hardcoding this as numpy can be ingested directly
+
+    source_instruction: str = "represent the document for retrieval"
+    query_instruction: str = (
+        "represent the document for retrieving the most similar documents"
+    )
+
+    @weak_lru(maxsize=1)
+    def ndims(self):
+        model = self.get_model()
+        return model.encode("foo").shape[0]
+
+    def compute_query_embeddings(self, query: str, *args, **kwargs) -> List[np.array]:
+        return self.generate_embeddings([[self.query_instruction, query]])
+
+    def compute_source_embeddings(self, texts: TEXT, *args, **kwargs) -> List[np.array]:
+        texts = self.sanitize_input(texts)
+        texts_formatted = []
+        for text in texts:
+            texts_formatted.append([self.source_instruction, text])
+        return self.generate_embeddings(texts_formatted)
+
+    def generate_embeddings(self, texts: List) -> List:
+        model = self.get_model()
+        res = model.encode(
+            texts,
+            batch_size=self.batch_size,
+            show_progress_bar=self.show_progress_bar,
+            normalize_embeddings=self.normalize_embeddings,
+        ).tolist()
+        return res
+
+    @weak_lru(maxsize=1)
+    def get_model(self):
+        instructor_embedding = self.safe_import(
+            "InstructorEmbedding", "InstructorEmbedding"
+        )
+        torch = self.safe_import("torch", "torch")
+
+        model = instructor_embedding.INSTRUCTOR(self.name)
+        if self.quantize:
+            if (
+                "qnnpack" in torch.backends.quantized.supported_engines
+            ):  # fix for https://github.com/pytorch/pytorch/issues/29327
+                torch.backends.quantized.engine = "qnnpack"
+            model = torch.quantization.quantize_dynamic(
+                model, {torch.nn.Linear}, dtype=torch.qint8
+            )
+        return model
--- a/python/lancedb/embeddings/open_clip.py
+++ b/python/lancedb/embeddings/open_clip.py
@@ -1,3 +1,15 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 import concurrent.futures
 import io
 import os
--- a/python/lancedb/embeddings/openai.py
+++ b/python/lancedb/embeddings/openai.py
@@ -1,3 +1,15 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 from typing import List, Union

 import numpy as np
--- a/python/lancedb/embeddings/sentence_transformers.py
+++ b/python/lancedb/embeddings/sentence_transformers.py
@@ -1,3 +1,15 @@
+#  Copyright (c) 2023. LanceDB Developers
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
 from typing import List, Union

 import numpy as np
@@ -5,6 +17,7 @@ from cachetools import cached

 from .base import TextEmbeddingFunction
 from .registry import register
+from .utils import weak_lru


@register("sentence-transformers")
@@ -30,7 +43,7 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
        name and device. This is cached so that the model is only loaded
        once per process.
        """
-        return self.__class__.get_embedding_model(self.name, self.device)
+        return self.get_embedding_model()

    def ndims(self):
        if self._ndims is None:
@@ -54,9 +67,8 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
            normalize_embeddings=self.normalize,
        ).tolist()

-    @classmethod
-    @cached(cache={})
-    def get_embedding_model(cls, name, device):
+    @weak_lru(maxsize=1)
+    def get_embedding_model(self):
        """
        Get the sentence-transformers embedding model specified by the
        name and device. This is cached so that the model is only loaded
@@ -71,7 +83,7 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):

        TODO: use lru_cache instead with a reasonable/configurable maxsize
        """
-        sentence_transformers = cls.safe_import(
+        sentence_transformers = self.safe_import(
            "sentence_transformers", "sentence-transformers"
        )
-        return sentence_transformers.SentenceTransformer(name, device=device)
+        return sentence_transformers.SentenceTransformer(self.name, device=self.device)
--- a/python/lancedb/embeddings/utils.py
+++ b/python/lancedb/embeddings/utils.py
@@ -11,10 +11,14 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

+import functools
 import math
+import random
 import socket
 import sys
+import time
 import urllib.error
+import weakref
 from typing import Callable, List, Union

 import numpy as np
@@ -162,6 +166,99 @@ class FunctionWrapper:
            yield from _chunker(arr)


+def weak_lru(maxsize=128):
+    """
+    LRU cache that keeps weak references to the objects it caches. Only caches the latest instance of the objects to make sure memory usage
+    is bounded.
+
+    Parameters
+    ----------
+    maxsize : int, default 128
+        The maximum number of objects to cache.
+
+    Returns
+    -------
+    Callable
+        A decorator that can be applied to a method.
+
+    Examples
+    --------
+    >>> class Foo:
+    ...     @weak_lru()
+    ...     def bar(self, x):
+    ...         return x
+    >>> foo = Foo()
+    >>> foo.bar(1)
+    1
+    >>> foo.bar(2)
+    2
+    >>> foo.bar(1)
+    1
+    """
+
+    def wrapper(func):
+        @functools.lru_cache(maxsize)
+        def _func(_self, *args, **kwargs):
+            return func(_self(), *args, **kwargs)
+
+        @functools.wraps(func)
+        def inner(self, *args, **kwargs):
+            return _func(weakref.ref(self), *args, **kwargs)
+
+        return inner
+
+    return wrapper
+
+
+def retry_with_exponential_backoff(
+    func,
+    initial_delay: float = 1,
+    exponential_base: float = 2,
+    jitter: bool = True,
+    max_retries: int = 7,
+    # errors: tuple = (),
+):
+    """Retry a function with exponential backoff.
+
+    Args:
+        func (function): The function to be retried.
+        initial_delay (float): Initial delay in seconds (default is 1).
+        exponential_base (float): The base for exponential backoff (default is 2).
+        jitter (bool): Whether to add jitter to the delay (default is True).
+        max_retries (int): Maximum number of retries (default is 10).
+        errors (tuple): Tuple of specific exceptions to retry on (default is (openai.error.RateLimitError,)).
+
+    Returns:
+        function: The decorated function.
+    """
+
+    def wrapper(*args, **kwargs):
+        num_retries = 0
+        delay = initial_delay
+
+        # Loop until a successful response or max_retries is hit or an exception is raised
+        while True:
+            try:
+                return func(*args, **kwargs)
+
+            # Currently retrying on all exceptions as there is no way to know the format of the error msgs used by different APIs
+            # We'll log the error and say that it is assumed that if this portion errors out, it's due to rate limit but the user
+            # should check the error message to be sure
+            except Exception as e:
+                num_retries += 1
+
+                if num_retries > max_retries:
+                    raise Exception(
+                        f"Maximum number of retries ({max_retries}) exceeded."
+                    )
+
+                delay *= exponential_base * (1 + jitter * random.random())
+                LOGGER.info(f"Retrying in {delay:.2f} seconds due to {e}")
+                time.sleep(delay)
+
+    return wrapper
+
+
 def url_retrieve(url: str):
    """
    Parameters
--- a/python/lancedb/query.py
+++ b/python/lancedb/query.py
@@ -14,7 +14,7 @@
 from __future__ import annotations

 from abc import ABC, abstractmethod
-from typing import List, Literal, Optional, Type, Union
+from typing import TYPE_CHECKING, List, Literal, Optional, Type, Union

 import deprecation
 import numpy as np
@@ -23,14 +23,49 @@ import pydantic

 from . import __version__
 from .common import VECTOR_COLUMN_NAME
-from .pydantic import LanceModel
 from .util import safe_import_pandas

+if TYPE_CHECKING:
+    from .pydantic import LanceModel
+
 pd = safe_import_pandas()


 class Query(pydantic.BaseModel):
-    """A Query"""
+    """The LanceDB Query
+
+    Attributes
+    ----------
+    vector : List[float]
+        the vector to search for
+    filter : Optional[str]
+        sql filter to refine the query with, optional
+    prefilter : bool
+        if True then apply the filter before vector search
+    k : int
+        top k results to return
+    metric : str
+        the distance metric between a pair of vectors,
+
+        can support L2 (default), Cosine and Dot.
+        [metric definitions][search]
+    columns : Optional[List[str]]
+        which columns to return in the results
+    nprobes : int
+        The number of probes used - optional
+
+        - A higher number makes search more accurate but also slower.
+
+        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+          tuning advice.
+    refine_factor : Optional[int]
+        Refine the results by reading extra elements and re-ranking them in memory - optional
+
+        - A higher number makes search more accurate but also slower.
+
+        - See discussion in [Querying an ANN Index][querying-an-ann-index] for
+          tuning advice.
+    """

    vector_column: str = VECTOR_COLUMN_NAME

@@ -61,6 +96,10 @@ class Query(pydantic.BaseModel):


 class LanceQueryBuilder(ABC):
+    """Build LanceDB query based on specific query type:
+    vector or full text search.
+    """
+
    @classmethod
    def create(
        cls,
@@ -103,7 +142,7 @@ class LanceQueryBuilder(ABC):
            if not isinstance(query, (list, np.ndarray)):
                conf = table.embedding_functions.get(vector_column_name)
                if conf is not None:
-                    query = conf.function.compute_query_embeddings(query)[0]
+                    query = conf.function.compute_query_embeddings_with_retry(query)[0]
                else:
                    msg = f"No embedding function for {vector_column_name}"
                    raise ValueError(msg)
@@ -114,7 +153,7 @@ class LanceQueryBuilder(ABC):
            else:
                conf = table.embedding_functions.get(vector_column_name)
                if conf is not None:
-                    query = conf.function.compute_query_embeddings(query)[0]
+                    query = conf.function.compute_query_embeddings_with_retry(query)[0]
                    return query, "vector"
                else:
                    return query, "fts"
@@ -133,11 +172,11 @@ class LanceQueryBuilder(ABC):
        deprecated_in="0.3.1",
        removed_in="0.4.0",
        current_version=__version__,
-        details="Use the bar function instead",
+        details="Use to_pandas() instead",
    )
    def to_df(self) -> "pd.DataFrame":
        """
-        Deprecated alias for `to_pandas()`. Please use `to_pandas()` instead.
+        *Deprecated alias for `to_pandas()`. Please use `to_pandas()` instead.*

        Execute the query and return the results as a pandas DataFrame.
        In addition to the selected columns, LanceDB also returns a vector
@@ -226,13 +265,20 @@ class LanceQueryBuilder(ABC):
        self._columns = columns
        return self

-    def where(self, where) -> LanceQueryBuilder:
+    def where(self, where: str, prefilter: bool = False) -> LanceQueryBuilder:
        """Set the where clause.

        Parameters
        ----------
        where: str
-            The where clause.
+            The where clause which is a valid SQL where clause. See
+            `Lance filter pushdown <https://lancedb.github.io/lance/read_and_write.html#filter-push-down>`_
+            for valid SQL expressions.
+        prefilter: bool, default False
+            If True, apply the filter before vector search, otherwise the
+            filter is applied on the result of vector search.
+            This feature is **EXPERIMENTAL** and may be removed and modified
+            without warning in the future.

        Returns
        -------
@@ -240,13 +286,12 @@ class LanceQueryBuilder(ABC):
            The LanceQueryBuilder object.
        """
        self._where = where
+        self._prefilter = prefilter
        return self


 class LanceVectorQueryBuilder(LanceQueryBuilder):
    """
-    A builder for nearest neighbor queries for LanceDB.
-
    Examples
    --------
    >>> import lancedb
@@ -302,7 +347,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
        Higher values will yield better recall (more likely to find vectors if
        they exist) at the expense of latency.

-        See discussion in [Querying an ANN Index][../querying-an-ann-index] for
+        See discussion in [Querying an ANN Index][querying-an-ann-index] for
        tuning advice.

        Parameters
@@ -369,14 +414,14 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
        Parameters
        ----------
        where: str
-            The where clause.
+            The where clause which is a valid SQL where clause. See
+            `Lance filter pushdown <https://lancedb.github.io/lance/read_and_write.html#filter-push-down>`_
+            for valid SQL expressions.
        prefilter: bool, default False
            If True, apply the filter before vector search, otherwise the
            filter is applied on the result of vector search.
            This feature is **EXPERIMENTAL** and may be removed and modified
-            without warning in the future. Currently this is only supported
-            in OSS and can only be used with a table that does not have an ANN
-            index.
+            without warning in the future.

        Returns
        -------
@@ -389,6 +434,8 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):


 class LanceFtsQueryBuilder(LanceQueryBuilder):
+    """A builder for full text search for LanceDB."""
+
    def __init__(self, table: "lancedb.table.Table", query: str):
        super().__init__(table)
        self._query = query
--- a/python/lancedb/remote/client.py
+++ b/python/lancedb/remote/client.py
@@ -13,7 +13,7 @@


 import functools
-from typing import Any, Callable, Dict, Optional, Union
+from typing import Any, Callable, Dict, Iterable, Optional, Union

 import aiohttp
 import attrs
@@ -151,15 +151,14 @@ class RestfulLanceDBClient:
            return await deserialize(resp)

    @_check_not_closed
-    async def list_tables(self, limit: int, page_token: str):
+    async def list_tables(
+        self, limit: int, page_token: Optional[str] = None
+    ) -> Iterable[str]:
        """List all tables in the database."""
-        try:
-            json = await self.get(
-                "/v1/table/", {"limit": limit, "page_token": page_token}
-            )
+        if page_token is None:
+            page_token = ""
+        json = await self.get("/v1/table/", {"limit": limit, "page_token": page_token})
        return json["tables"]
-        except StopAsyncIteration:
-            return []

    @_check_not_closed
    async def query(self, table_name: str, query: VectorQuery) -> VectorQueryResult:
--- a/python/lancedb/remote/db.py
+++ b/python/lancedb/remote/db.py
@@ -12,14 +12,19 @@
 #  limitations under the License.

 import asyncio
+import inspect
+import logging
 import uuid
-from typing import Iterator, Optional
+from typing import Iterable, List, Optional, Union
 from urllib.parse import urlparse

 import pyarrow as pa
+from overrides import override

 from ..common import DATA
 from ..db import DBConnection
+from ..embeddings import EmbeddingFunctionConfig
+from ..pydantic import LanceModel
 from ..table import Table, _sanitize_data
 from .arrow import to_ipc_binary
 from .client import ARROW_STREAM_CONTENT_TYPE, RestfulLanceDBClient
@@ -52,11 +57,13 @@ class RemoteDBConnection(DBConnection):
    def __repr__(self) -> str:
        return f"RemoveConnect(name={self.db_name})"

-    def table_names(self, last_token: str, limit=10) -> Iterator[str]:
+    @override
+    def table_names(self, page_token: Optional[str] = None, limit=10) -> Iterable[str]:
        """List the names of all tables in the database.
+
        Parameters
        ----------
-        last_token: str
+        page_token: str
            The last token to start the new page.

        Returns
@@ -65,15 +72,16 @@ class RemoteDBConnection(DBConnection):
        """
        while True:
            result = self._loop.run_until_complete(
-                self._client.list_tables(limit, last_token)
+                self._client.list_tables(limit, page_token)
            )
            if len(result) > 0:
-                last_token = result[len(result) - 1]
+                page_token = result[len(result) - 1]
            else:
                break
            for item in result:
-                yield result
+                yield item

+    @override
    def open_table(self, name: str) -> Table:
        """Open a Lance Table in the database.

@@ -88,23 +96,50 @@ class RemoteDBConnection(DBConnection):
        """
        from .table import RemoteTable

-        # TODO: check if table exists
-
+        # check if table exists
+        try:
+            self._loop.run_until_complete(
+                self._client.post(f"/v1/table/{name}/describe/")
+            )
+        except Exception:
+            logging.error(
+                "Table {name} does not exist."
+                "Please first call db.create_table({name}, data)"
+            )
        return RemoteTable(self, name)

+    @override
    def create_table(
        self,
        name: str,
        data: DATA = None,
-        schema: pa.Schema = None,
+        schema: Optional[Union[pa.Schema, LanceModel]] = None,
        on_bad_vectors: str = "error",
        fill_value: float = 0.0,
+        embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
    ) -> Table:
        if data is None and schema is None:
            raise ValueError("Either data or schema must be provided.")
+        if embedding_functions is not None:
+            raise NotImplementedError(
+                "embedding_functions is not supported for remote databases."
+                "Please vote https://github.com/lancedb/lancedb/issues/626 "
+                "for this feature."
+            )
+
+        if inspect.isclass(schema) and issubclass(schema, LanceModel):
+            # convert LanceModel to pyarrow schema
+            # note that it's possible this contains
+            # embedding function metadata already
+            schema = schema.to_arrow_schema()
+
        if data is not None:
            data = _sanitize_data(
-                data, schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+                data,
+                schema,
+                metadata=None,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
            )
        else:
            if schema is None:
@@ -126,6 +161,7 @@ class RemoteDBConnection(DBConnection):
        )
        return RemoteTable(self, name)

+    @override
    def drop_table(self, name: str):
        """Drop a table from the database.

--- a/python/lancedb/remote/table.py
+++ b/python/lancedb/remote/table.py
@@ -44,6 +44,14 @@ class RemoteTable(Table):
        schema = json_to_schema(resp["schema"])
        return schema

+    @property
+    def version(self) -> int:
+        """Get the current version of the table"""
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/describe/")
+        )
+        return resp["version"]
+
    def to_arrow(self) -> pa.Table:
        """Return the table as an Arrow table."""
        raise NotImplementedError("to_arrow() is not supported on the LanceDB cloud")
@@ -63,8 +71,62 @@ class RemoteTable(Table):
        vector_column_name: str = VECTOR_COLUMN_NAME,
        replace: bool = True,
        accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
    ):
-        raise NotImplementedError
+        """Create an index on the table.
+        Currently, the only parameters that matter are
+        the metric and the vector column name.
+
+        Parameters
+        ----------
+        metric : str
+            The metric to use for the index. Default is "L2".
+        num_partitions : int
+            The number of partitions to use for the index. Default is 256.
+        num_sub_vectors : int
+            The number of sub-vectors to use for the index. Default is 96.
+        vector_column_name : str
+            The name of the vector column. Default is "vector".
+        replace : bool
+            Whether to replace the existing index. Default is True.
+        accelerator : str, optional
+            If set, use the given accelerator to create the index.
+            Default is None. Currently not supported.
+        index_cache_size : int, optional
+            The size of the index cache in number of entries. Default value is 256.
+
+        Examples
+        --------
+        import lancedb
+        import uuid
+        from lancedb.schema import vector
+        conn = lancedb.connect("db://...", api_key="...", region="...")
+        table_name = uuid.uuid4().hex
+        schema = pa.schema(
+            [
+                    pa.field("id", pa.uint32(), False),
+                    pa.field("vector", vector(128), False),
+                    pa.field("s", pa.string(), False),
+            ]
+        )
+        table = conn.create_table(
+            table_name,
+            schema=schema,
+        )
+        table.create_index()
+        """
+        index_type = "vector"
+
+        data = {
+            "column": vector_column_name,
+            "index_type": index_type,
+            "metric_type": metric,
+            "index_cache_size": index_cache_size,
+        }
+        resp = self._conn._loop.run_until_complete(
+            self._conn._client.post(f"/v1/table/{self._name}/create_index/", data=data)
+        )
+        return resp

    def add(
        self,
@@ -99,8 +161,6 @@ class RemoteTable(Table):
        return LanceVectorQueryBuilder(self, query, vector_column_name)

    def _execute_query(self, query: Query) -> pa.Table:
-        if query.prefilter:
-            raise NotImplementedError("Cloud support for prefiltering is coming soon")
        result = self._conn._client.query(self._name, query)
        return self._conn._loop.run_until_complete(result).to_arrow()

--- a/python/lancedb/table.py
+++ b/python/lancedb/table.py
@@ -16,16 +16,14 @@ from __future__ import annotations
 import inspect
 import os
 from abc import ABC, abstractmethod
-from datetime import timedelta
 from functools import cached_property
-from typing import Any, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union

 import lance
 import numpy as np
 import pyarrow as pa
 import pyarrow.compute as pc
 from lance import LanceDataset
-from lance.dataset import CleanupStats, ReaderLike
 from lance.vector import vec_to_table

 from .common import DATA, VEC, VECTOR_COLUMN_NAME
@@ -35,6 +33,12 @@ from .query import LanceQueryBuilder, Query
 from .util import fs_from_uri, safe_import_pandas
 from .utils.events import register_event

+if TYPE_CHECKING:
+    from datetime import timedelta
+
+    from lance.dataset import CleanupStats, ReaderLike
+
+
 pd = safe_import_pandas()


@@ -86,7 +90,9 @@ def _append_vector_col(data: pa.Table, metadata: dict, schema: Optional[pa.Schem
    for vector_column, conf in functions.items():
        func = conf.function
        if vector_column not in data.column_names:
-            col_data = func.compute_source_embeddings(data[conf.source_column])
+            col_data = func.compute_source_embeddings_with_retry(
+                data[conf.source_column]
+            )
            if schema is not None:
                dtype = schema.field(vector_column).type
            else:
@@ -149,13 +155,13 @@ class Table(ABC):
    @property
    @abstractmethod
    def schema(self) -> pa.Schema:
-        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#) of
-        this Table
+        """The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)
+        of this Table

        """
        raise NotImplementedError

-    def to_pandas(self):
+    def to_pandas(self) -> "pd.DataFrame":
        """Return the table as a pandas DataFrame.

        Returns
@@ -182,6 +188,7 @@ class Table(ABC):
        vector_column_name: str = VECTOR_COLUMN_NAME,
        replace: bool = True,
        accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
    ):
        """Create an index on the table.

@@ -191,20 +198,23 @@ class Table(ABC):
            The distance metric to use when creating the index.
            Valid values are "L2", "cosine", or "dot".
            L2 is euclidean distance.
-        num_partitions: int
+        num_partitions: int, default 256
            The number of IVF partitions to use when creating the index.
            Default is 256.
-        num_sub_vectors: int
+        num_sub_vectors: int, default 96
            The number of PQ sub-vectors to use when creating the index.
            Default is 96.
        vector_column_name: str, default "vector"
            The vector column name to create the index.
        replace: bool, default True
-            If True, replace the existing index if it exists.
-            If False, raise an error if duplicate index exists.
+            - If True, replace the existing index if it exists.
+
+            - If False, raise an error if duplicate index exists.
        accelerator: str, default None
            If set, use the given accelerator to create the index.
            Only support "cuda" for now.
+        index_cache_size : int, optional
+            The size of the index cache in number of entries. Default value is 256.
        """
        raise NotImplementedError

@@ -220,8 +230,14 @@ class Table(ABC):

        Parameters
        ----------
-        data: list-of-dict, dict, pd.DataFrame
-            The data to insert into the table.
+        data: DATA
+            The data to insert into the table. Acceptable types are:
+
+            - dict or list-of-dict
+
+            - pandas.DataFrame
+
+            - pyarrow.Table or pyarrow.RecordBatch
        mode: str
            The mode to use when writing the data. Valid values are
            "append" and "overwrite".
@@ -242,31 +258,70 @@ class Table(ABC):
        query_type: str = "auto",
    ) -> LanceQueryBuilder:
        """Create a search query to find the nearest neighbors
-        of the given query vector.
+        of the given query vector. We currently support [vector search][search]
+        and [full-text search][experimental-full-text-search].
+
+        All query options are defined in [Query][lancedb.query.Query].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data)
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector")
+        ...     .where("original_width > 1000", prefilter=True)
+        ...     .select(["caption", "original_width"])
+        ...     .limit(2)
+        ...     .to_pandas())
+          caption  original_width           vector  _distance
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996

        Parameters
        ----------
-        query: str, list, np.ndarray, PIL.Image.Image, default None
-            The query to search for. If None then
-            the select/where/limit clauses are applied to filter
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/where/limit clauses are applied to filter
            the table
-        vector_column_name: str, default "vector"
+        vector_column_name: str
            The name of the vector column to search.
-        query_type: str, default "auto"
-            "vector", "fts", or "auto"
-            If "auto" then the query type is inferred from the query;
-            If `query` is a list/np.ndarray then the query type is "vector";
-            If `query` is a PIL.Image.Image then either do vector search
+            *default "vector"*
+        query_type: str
+            *default "auto"*.
+            Acceptable types are: "vector", "fts", or "auto"
+
+            - If "auto" then the query type is inferred from the query;
+
+                - If `query` is a list/np.ndarray then the query type is
+                "vector";
+
+                - If `query` is a PIL.Image.Image then either do vector search,
                or raise an error if no corresponding embedding function is found.
-            If `query` is a string, then the query type is "vector" if the
+
+            - If `query` is a string, then the query type is "vector" if the
            table has embedding functions else the query type is "fts"

        Returns
        -------
        LanceQueryBuilder
            A query builder object representing the query.
-            Once executed, the query returns selected columns, the vector,
-            and also the "_distance" column which is the distance between the query
+            Once executed, the query returns
+
+            - selected columns
+
+            - the vector
+
+            - and also the "_distance" column which is the distance between the query
            vector and the returned vector.
        """
        raise NotImplementedError
@@ -285,14 +340,19 @@ class Table(ABC):
        Parameters
        ----------
        where: str
-            The SQL where clause to use when deleting rows. For example, 'x = 2'
-            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.
+            The SQL where clause to use when deleting rows.
+
+            - For example, 'x = 2' or 'x IN (1, 2, 3)'.
+
+            The filter must not be empty, or it will error.

        Examples
        --------
        >>> import lancedb
        >>> data = [
-        ...   {"x": 1, "vector": [1, 2]}, {"x": 2, "vector": [3, 4]}, {"x": 3, "vector": [5, 6]}
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
        ... ]
        >>> db = lancedb.connect("./.lancedb")
        >>> table = db.create_table("my_table", data)
@@ -377,7 +437,8 @@ class LanceTable(Table):
        --------
        >>> import lancedb
        >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table = db.create_table("my_table",
+        ...    [{"vector": [1.1, 0.9], "type": "vector"}])
        >>> table.version
        2
        >>> table.to_pandas()
@@ -424,7 +485,8 @@ class LanceTable(Table):
        --------
        >>> import lancedb
        >>> db = lancedb.connect("./.lancedb")
-        >>> table = db.create_table("my_table", [{"vector": [1.1, 0.9], "type": "vector"}])
+        >>> table = db.create_table("my_table", [
+        ...     {"vector": [1.1, 0.9], "type": "vector"}])
        >>> table.version
        2
        >>> table.to_pandas()
@@ -497,6 +559,7 @@ class LanceTable(Table):
        vector_column_name=VECTOR_COLUMN_NAME,
        replace: bool = True,
        accelerator: Optional[str] = None,
+        index_cache_size: Optional[int] = None,
    ):
        """Create an index on the table."""
        self._dataset.create_index(
@@ -507,6 +570,7 @@ class LanceTable(Table):
            num_sub_vectors=num_sub_vectors,
            replace=replace,
            accelerator=accelerator,
+            index_cache_size=index_cache_size,
        )
        self._reset_dataset()
        register_event("create_index")
@@ -669,14 +733,39 @@ class LanceTable(Table):
        query_type: str = "auto",
    ) -> LanceQueryBuilder:
        """Create a search query to find the nearest neighbors
-        of the given query vector.
+        of the given query vector. We currently support [vector search][search]
+        and [full-text search][search].
+
+        Examples
+        --------
+        >>> import lancedb
+        >>> db = lancedb.connect("./.lancedb")
+        >>> data = [
+        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},
+        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},
+        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}
+        ... ]
+        >>> table = db.create_table("my_table", data)
+        >>> query = [0.4, 1.4, 2.4]
+        >>> (table.search(query, vector_column_name="vector")
+        ...     .where("original_width > 1000", prefilter=True)
+        ...     .select(["caption", "original_width"])
+        ...     .limit(2)
+        ...     .to_pandas())
+          caption  original_width           vector  _distance
+        0     foo            2000  [0.5, 3.4, 1.3]   5.220000
+        1    test            3000  [0.3, 6.2, 2.6]  23.089996

        Parameters
        ----------
-        query: str, list, np.ndarray, a PIL Image or None
-            The query to search for. If None then
-            the select/where/limit clauses are applied to filter
-            the table
+        query: list/np.ndarray/str/PIL.Image.Image, default None
+            The targetted vector to search for.
+
+            - *default None*.
+            Acceptable types are: list, np.ndarray, PIL.Image.Image
+
+            - If None then the select/[where][sql]/limit clauses are applied
+            to filter the table
        vector_column_name: str, default "vector"
            The name of the vector column to search.
        query_type: str, default "auto"
@@ -685,7 +774,7 @@ class LanceTable(Table):
            If `query` is a list/np.ndarray then the query type is "vector";
            If `query` is a PIL.Image.Image then either do vector search
            or raise an error if no corresponding embedding function is found.
-            If the query is a string, then the query type is "vector" if the
+            If the `query` is a string, then the query type is "vector" if the
            table has embedding functions, else the query type is "fts"

        Returns
@@ -720,7 +809,9 @@ class LanceTable(Table):
        --------
        >>> import lancedb
        >>> data = [
-        ...   {"x": 1, "vector": [1, 2]}, {"x": 2, "vector": [3, 4]}, {"x": 3, "vector": [5, 6]}
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
        ... ]
        >>> db = lancedb.connect("./.lancedb")
        >>> table = db.create_table("my_table", data)
@@ -740,7 +831,8 @@ class LanceTable(Table):
            The data to insert into the table.
            At least one of `data` or `schema` must be provided.
        schema: pa.Schema or LanceModel, optional
-            The schema of the table. If not provided, the schema is inferred from the data.
+            The schema of the table. If not provided,
+            the schema is inferred from the data.
            At least one of `data` or `schema` must be provided.
        mode: str, default "create"
            The mode to use when writing the data. Valid values are
@@ -811,7 +903,8 @@ class LanceTable(Table):
        file_info = fs.get_file_info(path)
        if file_info.type != pa.fs.FileType.Directory:
            raise FileNotFoundError(
-                f"Table {name} does not exist. Please first call db.create_table({name}, data)"
+                f"Table {name} does not exist."
+                f"Please first call db.create_table({name}, data)"
            )
        return tbl

@@ -838,7 +931,9 @@ class LanceTable(Table):
        --------
        >>> import lancedb
        >>> data = [
-        ...   {"x": 1, "vector": [1, 2]}, {"x": 2, "vector": [3, 4]}, {"x": 3, "vector": [5, 6]}
+        ...    {"x": 1, "vector": [1, 2]},
+        ...    {"x": 2, "vector": [3, 4]},
+        ...    {"x": 3, "vector": [5, 6]}
        ... ]
        >>> db = lancedb.connect("./.lancedb")
        >>> table = db.create_table("my_table", data)
@@ -872,12 +967,6 @@ class LanceTable(Table):

    def _execute_query(self, query: Query) -> pa.Table:
        ds = self.to_lance()
-        if query.prefilter:
-            for idx in ds.list_indices():
-                if query.vector_column in idx["fields"]:
-                    raise NotImplementedError(
-                        "Prefiltering for indexed vector column is coming soon."
-                    )
        return ds.to_table(
            columns=query.columns,
            filter=query.filter,
@@ -1019,7 +1108,8 @@ def _sanitize_vector_column(
    # ChunkedArray is annoying to work with, so we combine chunks here
    vec_arr = data[vector_column_name].combine_chunks()
    if pa.types.is_list(data[vector_column_name].type):
-        # if it's a variable size list array we make sure the dimensions are all the same
+        # if it's a variable size list array,
+        # we make sure the dimensions are all the same
        has_jagged_ndims = len(vec_arr.values) % len(data) != 0
        if has_jagged_ndims:
            data = _sanitize_jagged(
--- a/python/lancedb/utils/sentry_log.py
+++ b/python/lancedb/utils/sentry_log.py
@@ -63,7 +63,8 @@ def set_sentry():
        """
        if "exc_info" in hint:
            exc_type, exc_value, tb = hint["exc_info"]
-            if "out of memory" in str(exc_value).lower():
+            ignored_errors = ["out of memory", "no space left on device", "testing"]
+            if any(error in str(exc_value).lower() for error in ignored_errors):
                return None

        if is_git_dir():
@@ -97,7 +98,7 @@ def set_sentry():
            dsn="https://c63ef8c64e05d1aa1a96513361f3ca2f@o4505950840946688.ingest.sentry.io/4505950933614592",
            debug=False,
            include_local_variables=False,
-            traces_sample_rate=1.0,
+            traces_sample_rate=0.5,
            environment="production",  # 'dev' or 'production'
            before_send=before_send,
            ignore_errors=[KeyboardInterrupt, FileNotFoundError, bdb.BdbQuit],
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -1,9 +1,9 @@
 [project]
 name = "lancedb"
-version = "0.3.2"
+version = "0.3.4"
 dependencies = [
    "deprecation",
-    "pylance==0.8.7",
+    "pylance==0.8.17",
    "ratelimiter~=1.0",
    "retry>=0.9.2",
    "tqdm>=4.1.0",
@@ -14,7 +14,8 @@ dependencies = [
    "cachetools",
    "pyyaml>=6.0",
    "click>=8.1.7",
-    "requests>=2.31.0"
+    "requests>=2.31.0",
+    "overrides>=0.7"
 ]
 description = "lancedb"
 authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -52,7 +53,7 @@ tests = ["pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "requests"]
 dev = ["ruff", "pre-commit", "black"]
 docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
 clip = ["torch", "pillow", "open-clip"]
-embeddings = ["openai", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere"]
+embeddings = ["openai", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "InstructorEmbedding"]

 [project.scripts]
 lancedb = "lancedb.cli.cli:cli"
@@ -64,6 +65,9 @@ build-backend = "setuptools.build_meta"
 [tool.isort]
 profile = "black"

+[tool.ruff]
+select = ["F", "E", "W", "I", "G", "TCH", "PERF"]
+
 [tool.pytest.ini_options]
 addopts = "--strict-markers"
 markers = [
--- a/python/tests/test_db.py
+++ b/python/tests/test_db.py
@@ -129,7 +129,7 @@ def test_ingest_iterator(tmp_path):
                [
                    PydanticSchema(vector=[3.1, 4.1], item="foo", price=10.0),
                    PydanticSchema(vector=[5.9, 26.5], item="bar", price=20.0),
-                ]
+                ],
                # TODO: test pydict separately. it is unique column number and names contraint
            ]

@@ -150,6 +150,21 @@ def test_ingest_iterator(tmp_path):
    run_tests(PydanticSchema)


+def test_table_names(tmp_path):
+    db = lancedb.connect(tmp_path)
+    data = pd.DataFrame(
+        {
+            "vector": [[3.1, 4.1], [5.9, 26.5]],
+            "item": ["foo", "bar"],
+            "price": [10.0, 20.0],
+        }
+    )
+    db.create_table("test2", data=data)
+    db.create_table("test1", data=data)
+    db.create_table("test3", data=data)
+    assert db.table_names() == ["test1", "test2", "test3"]
+
+
 def test_create_mode(tmp_path):
    db = lancedb.connect(tmp_path)
    data = pd.DataFrame(
@@ -286,4 +301,29 @@ def test_replace_index(tmp_path):
        num_partitions=2,
        num_sub_vectors=4,
        replace=True,
+        index_cache_size=10,
    )
+
+
+def test_prefilter_with_index(tmp_path):
+    db = lancedb.connect(uri=tmp_path)
+    data = [
+        {"vector": np.random.rand(128), "item": "foo", "price": float(i)}
+        for i in range(1000)
+    ]
+    sample_key = data[100]["vector"]
+    table = db.create_table(
+        "test",
+        data,
+    )
+    table.create_index(
+        num_partitions=2,
+        num_sub_vectors=4,
+    )
+    table = (
+        table.search(sample_key)
+        .where("price == 500", prefilter=True)
+        .limit(5)
+        .to_arrow()
+    )
+    assert table.num_rows == 1
--- a/python/tests/test_embeddings.py
+++ b/python/tests/test_embeddings.py
@@ -15,13 +15,16 @@ import sys
 import lance
 import numpy as np
 import pyarrow as pa
+import pytest

-from lancedb.conftest import MockTextEmbeddingFunction
+import lancedb
+from lancedb.conftest import MockRateLimitedEmbeddingFunction, MockTextEmbeddingFunction
 from lancedb.embeddings import (
    EmbeddingFunctionConfig,
    EmbeddingFunctionRegistry,
    with_embeddings,
 )
+from lancedb.pydantic import LanceModel, Vector


 def mock_embed_func(input_data):
@@ -83,3 +86,29 @@ def test_embedding_function(tmp_path):
    expected = func.compute_query_embeddings("hello world")

    assert np.allclose(actual, expected)
+
+
+def test_embedding_function_rate_limit(tmp_path):
+    def _get_schema_from_model(model):
+        class Schema(LanceModel):
+            text: str = model.SourceField()
+            vector: Vector(model.ndims()) = model.VectorField()
+
+        return Schema
+
+    db = lancedb.connect(tmp_path)
+    registry = EmbeddingFunctionRegistry.get_instance()
+    model = registry.get("test-rate-limited").create(max_retries=0)
+    schema = _get_schema_from_model(model)
+    table = db.create_table("test", schema=schema, mode="overwrite")
+    table.add([{"text": "hello world"}])
+    with pytest.raises(Exception):
+        table.add([{"text": "hello world"}])
+    assert len(table) == 1
+
+    model = registry.get("test-rate-limited").create()
+    schema = _get_schema_from_model(model)
+    table = db.create_table("test", schema=schema, mode="overwrite")
+    table.add([{"text": "hello world"}])
+    table.add([{"text": "hello world"}])
+    assert len(table) == 2
--- a/python/tests/test_embeddings_slow.py
+++ b/python/tests/test_embeddings_slow.py
@@ -32,8 +32,8 @@ from lancedb.pydantic import LanceModel, Vector
 def test_sentence_transformer(alias, tmp_path):
    db = lancedb.connect(tmp_path)
    registry = get_registry()
-    func = registry.get(alias).create()
-    func2 = registry.get(alias).create()
+    func = registry.get(alias).create(max_retries=0)
+    func2 = registry.get(alias).create(max_retries=0)

    class Words(LanceModel):
        text: str = func.SourceField()
@@ -150,7 +150,11 @@ def test_openclip(tmp_path):
    os.environ.get("COHERE_API_KEY") is None, reason="COHERE_API_KEY not set"
 )  # also skip if cohere not installed
 def test_cohere_embedding_function():
-    cohere = get_registry().get("cohere").create(name="embed-multilingual-v2.0")
+    cohere = (
+        get_registry()
+        .get("cohere")
+        .create(name="embed-multilingual-v2.0", max_retries=0)
+    )

    class TextModel(LanceModel):
        text: str = cohere.SourceField()
@@ -162,3 +166,19 @@ def test_cohere_embedding_function():

    tbl.add(df)
    assert len(tbl.to_pandas()["vector"][0]) == cohere.ndims()
+
+
+@pytest.mark.slow
+def test_instructor_embedding(tmp_path):
+    model = get_registry().get("instructor").create()
+
+    class TextModel(LanceModel):
+        text: str = model.SourceField()
+        vector: Vector(model.ndims()) = model.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect(tmp_path)
+    tbl = db.create_table("test", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == model.ndims()
--- a/python/tests/test_table.py
+++ b/python/tests/test_table.py
@@ -213,6 +213,7 @@ def test_create_index_method():
                num_sub_vectors=96,
                vector_column_name="vector",
                replace=True,
+                index_cache_size=256,
            )

            # Check that the _dataset.create_index method was called
@@ -225,6 +226,7 @@ def test_create_index_method():
                num_sub_vectors=96,
                replace=True,
                accelerator=None,
+                index_cache_size=256,
            )


--- a/rust/ffi/node/Cargo.toml
+++ b/rust/ffi/node/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "vectordb-node"
-version = "0.3.3"
+version = "0.3.8"
 description = "Serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
 edition = "2018"
@@ -19,6 +19,7 @@ once_cell = "1"
 futures = "0.3"
 half = { workspace = true }
 lance = { workspace = true }
+lance-index = { workspace = true }
 lance-linalg = { workspace = true }
 vectordb = { path = "../../vectordb" }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
--- a/rust/ffi/node/src/index/vector.rs
+++ b/rust/ffi/node/src/index/vector.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use lance::index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
+use lance_index::vector::{ivf::IvfBuildParams, pq::PQBuildParams};
 use lance_linalg::distance::MetricType;
 use neon::context::FunctionContext;
 use neon::prelude::*;
@@ -70,7 +70,6 @@ fn get_index_params_builder(
                .map(|mt| {
                    let metric_type = mt.unwrap();
                    index_builder.metric_type(metric_type);
-                    pq_params.metric_type = metric_type;
                });

            let num_partitions = obj.get_opt_usize(cx, "num_partitions")?;
--- a/rust/ffi/node/src/lib.rs
+++ b/rust/ffi/node/src/lib.rs
@@ -239,6 +239,8 @@ fn main(mut cx: ModuleContext) -> NeonResult<()> {
    cx.export_function("tableDelete", JsTable::js_delete)?;
    cx.export_function("tableCleanupOldVersions", JsTable::js_cleanup)?;
    cx.export_function("tableCompactFiles", JsTable::js_compact)?;
+    cx.export_function("tableListIndices", JsTable::js_list_indices)?;
+    cx.export_function("tableIndexStats", JsTable::js_index_stats)?;
    cx.export_function(
        "tableCreateVectorIndex",
        index::vector::table_create_vector_index,
--- a/rust/ffi/node/src/table.rs
+++ b/rust/ffi/node/src/table.rs
@@ -247,7 +247,7 @@ impl JsTable {
        }

        rt.spawn(async move {
-            let stats = table.compact_files(options).await;
+            let stats = table.compact_files(options, None).await;

            deferred.settle_with(&channel, move |mut cx| {
                let stats = stats.or_throw(&mut cx)?;
@@ -276,4 +276,91 @@ impl JsTable {
        });
        Ok(promise)
    }
+
+    pub(crate) fn js_list_indices(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let rt = runtime(&mut cx)?;
+        let (deferred, promise) = cx.promise();
+        // let predicate = cx.argument::<JsString>(0)?.value(&mut cx);
+        let channel = cx.channel();
+        let table = js_table.table.clone();
+
+        rt.spawn(async move {
+            let indices = table.load_indices().await;
+
+            deferred.settle_with(&channel, move |mut cx| {
+                let indices = indices.or_throw(&mut cx)?;
+
+                let output = JsArray::new(&mut cx, indices.len() as u32);
+                for (i, index) in indices.iter().enumerate() {
+                    let js_index = JsObject::new(&mut cx);
+                    let index_name = cx.string(index.index_name.clone());
+                    js_index.set(&mut cx, "name", index_name)?;
+
+                    let index_uuid = cx.string(index.index_uuid.clone());
+                    js_index.set(&mut cx, "uuid", index_uuid)?;
+
+                    let js_index_columns = JsArray::new(&mut cx, index.columns.len() as u32);
+                    for (j, column) in index.columns.iter().enumerate() {
+                        let js_column = cx.string(column.clone());
+                        js_index_columns.set(&mut cx, j as u32, js_column)?;
+                    }
+                    js_index.set(&mut cx, "columns", js_index_columns)?;
+
+                    output.set(&mut cx, i as u32, js_index)?;
+                }
+
+                Ok(output)
+            })
+        });
+        Ok(promise)
+    }
+
+    pub(crate) fn js_index_stats(mut cx: FunctionContext) -> JsResult<JsPromise> {
+        let js_table = cx.this().downcast_or_throw::<JsBox<JsTable>, _>(&mut cx)?;
+        let rt = runtime(&mut cx)?;
+        let (deferred, promise) = cx.promise();
+        let index_uuid = cx.argument::<JsString>(0)?.value(&mut cx);
+        let channel = cx.channel();
+        let table = js_table.table.clone();
+
+        rt.spawn(async move {
+            let load_stats = futures::try_join!(
+                table.count_indexed_rows(&index_uuid),
+                table.count_unindexed_rows(&index_uuid)
+            );
+
+            deferred.settle_with(&channel, move |mut cx| {
+                let (indexed_rows, unindexed_rows) = load_stats.or_throw(&mut cx)?;
+
+                let output = JsObject::new(&mut cx);
+
+                match indexed_rows {
+                    Some(x) => {
+                        let i = cx.number(x as f64);
+                        output.set(&mut cx, "numIndexedRows", i)?;
+                    }
+                    None => {
+                        let null = cx.null();
+                        output.set(&mut cx, "numIndexedRows", null)?;
+                    }
+                };
+
+                match unindexed_rows {
+                    Some(x) => {
+                        let i = cx.number(x as f64);
+                        output.set(&mut cx, "numUnindexedRows", i)?;
+                    }
+                    None => {
+                        let null = cx.null();
+                        output.set(&mut cx, "numUnindexedRows", null)?;
+                    }
+                };
+
+                Ok(output)
+            })
+        });
+
+        Ok(promise)
+    }
 }
--- a/rust/vectordb/Cargo.toml
+++ b/rust/vectordb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "vectordb"
-version = "0.3.3"
+version = "0.3.8"
 edition = "2021"
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license = "Apache-2.0"
@@ -21,6 +21,7 @@ object_store = { workspace = true }
 snafu = { workspace = true }
 half = { workspace = true }
 lance = { workspace = true }
+lance-index = { workspace = true }
 lance-linalg = { workspace = true }
 lance-testing = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread"] }
--- a/rust/vectordb/src/database.rs
+++ b/rust/vectordb/src/database.rs
@@ -161,7 +161,7 @@ impl Database {
    ///
    /// * A [Vec<String>] with all table names.
    pub async fn table_names(&self) -> Result<Vec<String>> {
-        let f = self
+        let mut f = self
            .object_store
            .read_dir(self.base_path.clone())
            .await?
@@ -175,7 +175,8 @@ impl Database {
                is_lance.unwrap_or(false)
            })
            .filter_map(|p| p.file_stem().and_then(|s| s.to_str().map(String::from)))
-            .collect();
+            .collect::<Vec<String>>();
+        f.sort();
        Ok(f)
    }

@@ -312,8 +313,8 @@ mod tests {
        let db = Database::connect(uri).await.unwrap();
        let tables = db.table_names().await.unwrap();
        assert_eq!(tables.len(), 2);
-        assert!(tables.contains(&String::from("table1")));
-        assert!(tables.contains(&String::from("table2")));
+        assert!(tables[0].eq(&String::from("table1")));
+        assert!(tables[1].eq(&String::from("table2")));
    }

    #[tokio::test]
--- a/rust/vectordb/src/index/vector.rs
+++ b/rust/vectordb/src/index/vector.rs
@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use lance::index::vector::ivf::IvfBuildParams;
+use lance::format::{Index, Manifest};
 use lance::index::vector::pq::PQBuildParams;
 use lance::index::vector::VectorIndexParams;
+use lance_index::vector::ivf::IvfBuildParams;
 use lance_linalg::distance::MetricType;

 pub trait VectorIndexBuilder {
@@ -98,7 +99,11 @@ impl VectorIndexBuilder for IvfPQIndexBuilder {
        let ivf_params = self.ivf_params.clone().unwrap_or_default();
        let pq_params = self.pq_params.clone().unwrap_or_default();

-        VectorIndexParams::with_ivf_pq_params(pq_params.metric_type, ivf_params, pq_params)
+        VectorIndexParams::with_ivf_pq_params(
+            self.metric_type.unwrap_or(MetricType::L2),
+            ivf_params,
+            pq_params,
+        )
    }

    fn get_replace(&self) -> bool {
@@ -106,13 +111,34 @@ impl VectorIndexBuilder for IvfPQIndexBuilder {
    }
 }

+pub struct VectorIndex {
+    pub columns: Vec<String>,
+    pub index_name: String,
+    pub index_uuid: String,
+}
+
+impl VectorIndex {
+    pub fn new_from_format(manifest: &Manifest, index: &Index) -> VectorIndex {
+        let fields = index
+            .fields
+            .iter()
+            .map(|i| manifest.schema.fields[*i as usize].name.clone())
+            .collect();
+        VectorIndex {
+            columns: fields,
+            index_name: index.name.clone(),
+            index_uuid: index.uuid.to_string(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;

-    use lance::index::vector::ivf::IvfBuildParams;
-    use lance::index::vector::pq::PQBuildParams;
    use lance::index::vector::StageParams;
+    use lance_index::vector::ivf::IvfBuildParams;
+    use lance_index::vector::pq::PQBuildParams;

    use crate::index::vector::{IvfPQIndexBuilder, VectorIndexBuilder};

@@ -158,7 +184,6 @@ mod tests {
        pq_params.max_iters = 1;
        pq_params.num_bits = 8;
        pq_params.num_sub_vectors = 50;
-        pq_params.metric_type = MetricType::Cosine;
        pq_params.max_opq_iters = 2;
        index_builder.ivf_params(ivf_params);
        index_builder.pq_params(pq_params);
@@ -176,7 +201,6 @@ mod tests {
            assert_eq!(pq_params.max_iters, 1);
            assert_eq!(pq_params.num_bits, 8);
            assert_eq!(pq_params.num_sub_vectors, 50);
-            assert_eq!(pq_params.metric_type, MetricType::Cosine);
            assert_eq!(pq_params.max_opq_iters, 2);
        } else {
            assert!(false, "Expected second stage to be pq")
--- a/rust/vectordb/src/io/object_store.rs
+++ b/rust/vectordb/src/io/object_store.rs
@@ -25,7 +25,8 @@ use bytes::Bytes;
 use futures::{stream::BoxStream, FutureExt, StreamExt};
 use lance::io::object_store::WrappingObjectStore;
 use object_store::{
-    path::Path, GetOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore, Result,
+    path::Path, Error, GetOptions, GetResult, ListResult, MultipartId, ObjectMeta, ObjectStore,
+    Result,
 };

 use async_trait::async_trait;
@@ -120,7 +121,10 @@ impl ObjectStore for MirroringObjectStore {

    async fn delete(&self, location: &Path) -> Result<()> {
        if !location.primary_only() {
-            self.secondary.delete(location).await?;
+            match self.secondary.delete(location).await {
+                Err(Error::NotFound { .. }) | Ok(_) => {}
+                Err(e) => return Err(e),
+            }
        }
        self.primary.delete(location).await
    }
--- a/rust/vectordb/src/table.rs
+++ b/rust/vectordb/src/table.rs
@@ -13,19 +13,23 @@
 // limitations under the License.

 use chrono::Duration;
+use lance::dataset::builder::DatasetBuilder;
+use lance_index::IndexType;
 use std::sync::Arc;

 use arrow_array::{Float32Array, RecordBatchReader};
 use arrow_schema::SchemaRef;
 use lance::dataset::cleanup::RemovalStats;
-use lance::dataset::optimize::{compact_files, CompactionMetrics, CompactionOptions};
+use lance::dataset::optimize::{
+    compact_files, CompactionMetrics, CompactionOptions, IndexRemapperOptions,
+};
 use lance::dataset::{Dataset, WriteParams};
-use lance::index::IndexType;
+use lance::index::DatasetIndexExt;
 use lance::io::object_store::WrappingObjectStore;
 use std::path::Path;

 use crate::error::{Error, Result};
-use crate::index::vector::VectorIndexBuilder;
+use crate::index::vector::{VectorIndex, VectorIndexBuilder};
 use crate::query::Query;
 use crate::utils::{PatchReadParam, PatchWriteParam};
 use crate::WriteMode;
@@ -94,7 +98,10 @@ impl Table {
            Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
            None => params,
        };
-        let dataset = Dataset::open_with_params(uri, &params)
+
+        let dataset = DatasetBuilder::from_uri(uri)
+            .with_read_params(params)
+            .load()
            .await
            .map_err(|e| match e {
                lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
@@ -238,8 +245,6 @@ impl Table {

    /// Create index on the table.
    pub async fn create_index(&mut self, index_builder: &impl VectorIndexBuilder) -> Result<()> {
-        use lance::index::DatasetIndexExt;
-
        let mut dataset = self.dataset.as_ref().clone();
        dataset
            .create_index(
@@ -257,6 +262,14 @@ impl Table {
        Ok(())
    }

+    pub async fn optimize_indices(&mut self) -> Result<()> {
+        let mut dataset = self.dataset.as_ref().clone();
+
+        dataset.optimize_indices().await?;
+
+        Ok(())
+    }
+
    /// Insert records into this Table
    ///
    /// # Arguments
@@ -353,12 +366,45 @@ impl Table {
    /// for faster reads.
    ///
    /// This calls into [lance::dataset::optimize::compact_files].
-    pub async fn compact_files(&mut self, options: CompactionOptions) -> Result<CompactionMetrics> {
+    pub async fn compact_files(
+        &mut self,
+        options: CompactionOptions,
+        remap_options: Option<Arc<dyn IndexRemapperOptions>>,
+    ) -> Result<CompactionMetrics> {
        let mut dataset = self.dataset.as_ref().clone();
-        let metrics = compact_files(&mut dataset, options, None).await?;
+        let metrics = compact_files(&mut dataset, options, remap_options).await?;
        self.dataset = Arc::new(dataset);
        Ok(metrics)
    }
+
+    pub fn count_fragments(&self) -> usize {
+        self.dataset.count_fragments()
+    }
+
+    pub async fn count_deleted_rows(&self) -> Result<usize> {
+        Ok(self.dataset.count_deleted_rows().await?)
+    }
+
+    pub async fn num_small_files(&self, max_rows_per_group: usize) -> usize {
+        self.dataset.num_small_files(max_rows_per_group).await
+    }
+
+    pub async fn count_indexed_rows(&self, index_uuid: &str) -> Result<Option<usize>> {
+        Ok(self.dataset.count_indexed_rows(index_uuid).await?)
+    }
+
+    pub async fn count_unindexed_rows(&self, index_uuid: &str) -> Result<Option<usize>> {
+        Ok(self.dataset.count_unindexed_rows(index_uuid).await?)
+    }
+
+    pub async fn load_indices(&self) -> Result<Vec<VectorIndex>> {
+        let (indices, mf) =
+            futures::try_join!(self.dataset.load_indices(), self.dataset.latest_manifest())?;
+        Ok(indices
+            .iter()
+            .map(|i| VectorIndex::new_from_format(&mf, i))
+            .collect())
+    }
 }

 #[cfg(test)]
@@ -373,9 +419,9 @@ mod tests {
    use arrow_data::ArrayDataBuilder;
    use arrow_schema::{DataType, Field, Schema};
    use lance::dataset::{Dataset, WriteMode};
-    use lance::index::vector::ivf::IvfBuildParams;
    use lance::index::vector::pq::PQBuildParams;
    use lance::io::object_store::{ObjectStoreParams, WrappingObjectStore};
+    use lance_index::vector::ivf::IvfBuildParams;
    use rand::Rng;
    use tempfile::tempdir;
Author	SHA1	Message	Date
Lance Release	38321fa226	[python] Bump version: 0.3.3 → 0.3.4	2023-11-19 00:24:01 +00:00
Lance Release	22749c3fa2	Updating package-lock.json	2023-11-19 00:04:08 +00:00
Lance Release	123a49df77	Bump version: 0.3.7 → 0.3.8	2023-11-19 00:03:58 +00:00
Will Jones	a57aa4b142	chore: upgrade lance to v0.8.17 (#656 ) Readying for the next Lance release.	2023-11-18 15:57:23 -08:00
Rok Mihevc	d8e3e54226	feat(python): expose index cache size (#655 ) This is to enable https://github.com/lancedb/lancedb/issues/641. Should be merged after https://github.com/lancedb/lance/pull/1587 is released.	2023-11-18 14:17:40 -08:00
Ayush Chaurasia	ccfdf4853a	[Docs]: Add Instructor embeddings and rate limit handler docs (#651 )	2023-11-18 06:08:26 +05:30
Ayush Chaurasia	87e5d86e90	[Docs][SEO] Add sitemap and robots.txt (#645 ) Sitemap improves SEO by ranking pages and tracking updates.	2023-11-18 06:08:13 +05:30
Aidan	1cf8a3e4e0	SaaS create_index API (#649 )	2023-11-15 19:12:52 -05:00
Lance Release	5372843281	Updating package-lock.json	2023-11-15 03:15:10 +00:00
Lance Release	54677b8f0b	Updating package-lock.json	2023-11-15 02:42:38 +00:00
Lance Release	ebcf9bf6ae	Bump version: 0.3.6 → 0.3.7	2023-11-15 02:42:25 +00:00
Bert	797514bcbf	fix: node remote implement table.countRows (#648 )	2023-11-13 17:43:20 -05:00
Rok Mihevc	1c872ce501	feat: add RemoteTable.version in Python (#644 ) Please note: this is not tested as we don't have a server here and testing against a mock object wouldn't be that interesting.	2023-11-13 21:43:48 +01:00
Bert	479f471c14	fix: node send db header for GET requests (#646 )	2023-11-11 16:33:25 -05:00
Ayush Chaurasia	ae0d2f2599	fix: Pydantic 1.x compat for weak_lru caching in embeddings API (#643 ) Colab has pydantic 1.x by default and pydantic 1.x BaseModel objects don't support weakref creation by default that we use to cache embedding models https://github.com/lancedb/lancedb/blob/main/python/lancedb/embeddings/utils.py#L206 . It needs to be added to slot.	2023-11-10 15:02:38 +05:30
Ayush Chaurasia	1e8678f11a	Multi-task instructor model with quantization support & weak_lru cache for embedding function models (#612 ) resolves #608	2023-11-09 12:34:18 +05:30
QianZhu	662968559d	fix saas open_table and table_names issues (#640 ) - added check whether a table exists in SaaS open_table - remove prefilter not supported warning in SaaS search - fixed issues for SaaS table_names	2023-11-07 17:34:38 -08:00
Rob Meng	9d895801f2	upgrade lance to 0.8.14 (#636 ) upgrade lance	2023-11-07 19:01:29 -05:00
Rob Meng	80613a40fd	skip missing file on mirrored dir when deleting (#635 ) mirrored store is not garueeteed to have all the files. Ignore the ones that doesn't exist.	2023-11-07 12:33:32 -05:00
Lei Xu	d43ef7f11e	chore: apple silicon runner (#633 ) Close #632	2023-11-06 21:04:32 -08:00
Lei Xu	554e068917	chore: improve create_table API consistency between local and remote SDK (#627 )	2023-11-03 13:15:11 -07:00
Bert	567734dd6e	fix: node remote connection handles non http errors (#624 ) https://github.com/lancedb/lancedb/issues/623 Fixes issue trying to print response status when using remote client. If the error is not an HTTP error (e.g. dns/network failure), there won't be a response.	2023-11-03 10:24:56 -04:00
Ayush Chaurasia	1589499f89	Exponential standoff retry support for handling rate limited embedding functions (#614 ) Users ingesting data using rate limited apis don't need to manually make the process sleep for counter rate limits resolves #579	2023-11-02 19:20:10 +05:30
Lance Release	682e95fa83	Updating package-lock.json	2023-11-01 22:20:49 +00:00
Lance Release	1ad5e7f2f0	Updating package-lock.json	2023-11-01 21:16:20 +00:00
Lance Release	ddb3ef4ce5	Bump version: 0.3.5 → 0.3.6	2023-11-01 21:16:06 +00:00
Lance Release	ef20b2a138	[python] Bump version: 0.3.2 → 0.3.3	2023-11-01 21:15:55 +00:00
Lei Xu	2e0f251bfd	chore: bump lance to 8.10 (#622 )	2023-11-01 14:14:38 -07:00
Ayush Chaurasia	2cb91e818d	Disable posthog on docs & reduce sentry trace factor (#607 ) - posthog charges per event and docs events are registered very frequently. We can keep tracking them on GA - Reduced sentry trace factor	2023-11-02 01:13:16 +05:30
Chang She	2835c76336	doc: node sdk now supports windows (#616 )	2023-11-01 10:04:18 -07:00
Bert	8068a2bbc3	ci: cancel in progress runs on new push (#620 )	2023-11-01 11:33:48 -04:00
Bert	24111d543a	fix!: sort table names (#619 ) https://github.com/lancedb/lance/issues/1385	2023-11-01 10:50:09 -04:00
QianZhu	7eec2b8f9a	Qian/query option doc (#615 ) - API documentation improvement for queries (table.search) - a small bug fix for the remote API on create_table ![image](https://github.com/lancedb/lancedb/assets/1305083/712e9bd3-deb8-4d81-8cd0-d8e98ef68f4e) ![image](https://github.com/lancedb/lancedb/assets/1305083/ba22125a-8c36-4e34-a07f-e39f0136e62c)	2023-10-31 19:50:05 -07:00
Will Jones	b2b70ea399	increment pylance (#618 )	2023-10-31 18:07:03 -07:00
Bert	e50a3c1783	added api docs for prefilter flag (#617 ) Added the prefilter flag argument to the `LanceQueryBuilder.where`. This should make it display here: https://lancedb.github.io/lancedb/python/python/#lancedb.query.LanceQueryBuilder.select And also in intellisense like this: <img width="848" alt="image" src="https://github.com/lancedb/lancedb/assets/5846846/e0c53f4f-96bc-411b-9159-680a6c4d0070"> Also adds some improved documentation about the `where` argument to this method. --------- Co-authored-by: Weston Pace <weston.pace@gmail.com>	2023-10-31 16:39:32 -04:00
Weston Pace	b517134309	feat: allow prefiltering with index (#610 ) Support for prefiltering with an index was added in lance version 0.8.7. We can remove the lancedb check that prevents this. Closes #261	2023-10-31 13:11:03 -07:00
Lei Xu	6fb539b5bf	doc: add doc to use GPU for indexing (#611 )	2023-10-30 15:25:00 -07:00
Lance Release	f37fe120fd	Updating package-lock.json	2023-10-26 22:30:16 +00:00
Lance Release	2e115acb9a	Updating package-lock.json	2023-10-26 21:48:01 +00:00
Lance Release	27a638362d	Bump version: 0.3.4 → 0.3.5	2023-10-26 21:47:44 +00:00
Bert	22a6695d7a	fix conv version (#605 )	2023-10-26 17:44:11 -04:00
Lance Release	57eff82ee7	Updating package-lock.json	2023-10-26 21:03:07 +00:00
Lance Release	7732f7d41c	Bump version: 0.3.3 → 0.3.4	2023-10-26 21:02:52 +00:00
Bert	5ca98c326f	feat: added dataset stats api to node (#604 )	2023-10-26 17:00:48 -04:00
Bert	b55db397eb	feat: added data stats apis (#596 )	2023-10-26 13:10:17 -04:00
Rob Meng	c04d72ac8a	expose remap index api (#603 ) expose index remap options in `compact_files`	2023-10-25 22:10:37 -04:00
Rob Meng	28b02fb72a	feat: expose optimize index api (#602 ) expose `optimize_index` api.	2023-10-25 19:40:23 -04:00