feat: connection pool for sync client

2025-12-25 14:29:56 +00:00 · 2024-11-25 14:20:37 -05:00
42 changed files with 238 additions and 445 deletions
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.14.0-beta.0"
+current_version = "0.13.1-beta.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/.github/workflows/npm-publish.yml
+++ b/.github/workflows/npm-publish.yml
@@ -133,7 +133,7 @@ jobs:
          free -h
      - name: Build Linux Artifacts
        run: |
-          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-gnu
+          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
@@ -143,7 +143,7 @@ jobs:

  node-linux-musl:
    name: vectordb (${{ matrix.config.arch}}-unknown-linux-musl)
-    runs-on: ${{ matrix.config.runner }}
+    runs-on: ubuntu-latest
    container: alpine:edge
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
@@ -152,10 +152,7 @@ jobs:
      matrix:
        config:
          - arch: x86_64
-            runner: ubuntu-latest
          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
    steps:
      - name: Checkout
        uses: actions/checkout@v4
@@ -188,7 +185,7 @@ jobs:
      - name: Build Linux Artifacts
        run: |
          source ./saved_env
-          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }} ${{ matrix.config.arch }}-unknown-linux-musl
+          bash ci/manylinux_node/build_vectordb.sh ${{ matrix.config.arch }}
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
@@ -249,7 +246,7 @@ jobs:

  nodejs-linux-musl:
    name: lancedb (${{ matrix.config.arch}}-unknown-linux-musl
-    runs-on: ${{ matrix.config.runner }}
+    runs-on: ubuntu-latest
    container: alpine:edge
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
@@ -258,10 +255,7 @@ jobs:
      matrix:
        config:
          - arch: x86_64
-            runner: ubuntu-latest
          - arch: aarch64
-            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
-            runner: buildjet-16vcpu-ubuntu-2204-arm
    steps:
      - name: Checkout
        uses: actions/checkout@v4
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -23,14 +23,13 @@ rust-version = "1.80.0"                                                     # TO
 [workspace.dependencies]
 lance = { "version" = "=0.20.0", "features" = [
    "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-io = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-index = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-linalg = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-table = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-testing = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-datafusion = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
-lance-encoding = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.3" }
+], git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-index = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-linalg = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-table = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-testing = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-datafusion = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
+lance-encoding = { version = "=0.20.0", git = "https://github.com/lancedb/lance.git", tag = "v0.20.0-beta.2" }
 # Note that this one does not include pyarrow
 arrow = { version = "52.2", optional = false }
 arrow-array = "52.2"
--- a/ci/build_linux_artifacts.sh
+++ b/ci/build_linux_artifacts.sh
@@ -1,9 +1,8 @@
 #!/bin/bash
 set -e
 ARCH=${1:-x86_64}
-TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}

-# We pass down the current user so that when we later mount the local files
+# We pass down the current user so that when we later mount the local files 
 # into the container, the files are accessible by the current user.
 pushd ci/manylinux_node
 docker build \
@@ -19,4 +18,4 @@ docker run \
    -v $(pwd):/io -w /io \
    --memory-swap=-1 \
    lancedb-node-manylinux \
-    bash ci/manylinux_node/build_vectordb.sh $ARCH $TARGET_TRIPLE
+    bash ci/manylinux_node/build_vectordb.sh $ARCH
--- a/ci/manylinux_node/build_vectordb.sh
+++ b/ci/manylinux_node/build_vectordb.sh
@@ -2,7 +2,6 @@
 # Builds the node module for manylinux. Invoked by ci/build_linux_artifacts.sh.
 set -e
 ARCH=${1:-x86_64}
-TARGET_TRIPLE=${2:-x86_64-unknown-linux-gnu}

 if [ "$ARCH" = "x86_64" ]; then
    export OPENSSL_LIB_DIR=/usr/local/lib64/
@@ -18,4 +17,4 @@ FILE=$HOME/.bashrc && test -f $FILE && source $FILE
 cd node
 npm ci
 npm run build-release
-npm run pack-build -t $TARGET_TRIPLE
+npm run pack-build
--- a/docs/mkdocs.yml
+++ b/docs/mkdocs.yml
@@ -55,9 +55,6 @@ plugins:
            show_signature_annotations: true
            show_root_heading: true
            members_order: source
-            docstring_section_style: list
-            signature_crossrefs: true
-            separate_signature: true
          import:
            # for cross references
            - https://arrow.apache.org/docs/objects.inv
--- a/docs/src/guides/scalar_index.md
+++ b/docs/src/guides/scalar_index.md
@@ -1,35 +1,23 @@
-# Building a Scalar Index
+# Building Scalar Index

-Scalar indices organize data by scalar attributes (e.g. numbers, categorical values), enabling fast filtering of vector data. In vector databases, scalar indices accelerate the retrieval of scalar data associated with vectors, thus enhancing the query performance when searching for vectors that meet certain scalar criteria. 
-
-Similar to many SQL databases, LanceDB supports several types of scalar indices to accelerate search
+Similar to many SQL databases, LanceDB supports several types of Scalar indices to accelerate search
 over scalar columns.

- `BTREE`: The most common type is BTREE. The index stores a copy of the
-  column in sorted order. This sorted copy allows a binary search to be used to
-  satisfy queries.
- `BITMAP`: this index stores a bitmap for each unique value in the column. It 
-  uses a series of bits to indicate whether a value is present in a row of a table
- `LABEL_LIST`: a special index that can be used on `List<T>` columns to
-  support queries with `array_contains_all` and `array_contains_any`
-  using an underlying bitmap index.
+- `BTREE`: The most common type is BTREE. This index is inspired by the btree data structure
+  although only the first few layers of the btree are cached in memory.
+  It will perform well on columns with a large number of unique values and few rows per value.
+- `BITMAP`: this index stores a bitmap for each unique value in the column.
+  This index is useful for columns with a finite number of unique values and many rows per value.
+  For example, columns that represent "categories", "labels", or "tags"
+- `LABEL_LIST`: a special index that is used to index list columns whose values have a finite set of possibilities.
  For example, a column that contains lists of tags (e.g. `["tag1", "tag2", "tag3"]`) can be indexed with a `LABEL_LIST` index.

-!!! tips "How to choose the right scalar index type"
-
-    `BTREE`: This index is good for scalar columns with mostly distinct values and does best when the query is highly selective.
-    
-    `BITMAP`: This index works best for low-cardinality numeric or string columns, where the number of unique values is small (i.e., less than a few thousands).
-    
-    `LABEL_LIST`: This index should be used for columns containing list-type data.
-
 | Data Type                                                       | Filter                                    | Index Type   |
 | --------------------------------------------------------------- | ----------------------------------------- | ------------ |
 | Numeric, String, Temporal                                       | `<`, `=`, `>`, `in`, `between`, `is null` | `BTREE`      |
 | Boolean, numbers or strings with fewer than 1,000 unique values | `<`, `=`, `>`, `in`, `between`, `is null` | `BITMAP`     |
 | List of low cardinality of numbers or strings                   | `array_has_any`, `array_has_all`          | `LABEL_LIST` |

-### Create a scalar index
 === "Python"

    ```python
@@ -58,7 +46,7 @@ over scalar columns.
        await tlb.create_index("publisher", { config: lancedb.Index.bitmap() })
        ```

-The following scan will be faster if the column `book_id` has a scalar index:
+For example, the following scan will be faster if the column `my_col` has a scalar index:

 === "Python"

@@ -118,30 +106,3 @@ Scalar indices can also speed up scans containing a vector search or full text s
          .limit(10)
          .toArray();
        ```
-### Update a scalar index
-Updating the table data (adding, deleting, or modifying records) requires that you also update the scalar index. This can be done by calling `optimize`, which will trigger an update to the existing scalar index.
-=== "Python"
-
-    ```python
-    table.add([{"vector": [7, 8], "book_id": 4}])
-    table.optimize()
-    ```
-
-=== "TypeScript"
-
-    ```typescript
-    await tbl.add([{ vector: [7, 8], book_id: 4 }]);
-    await tbl.optimize();
-    ```
-
-=== "Rust"
-
-    ```rust
-    let more_data: Box<dyn RecordBatchReader + Send> = create_some_records()?;
-    tbl.add(more_data).execute().await?;
-    tbl.optimize(OptimizeAction::All).execute().await?;
-    ```
-
-!!! note
-
-    New data added after creating the scalar index will still appear in search results if optimize is not used, but with increased latency due to a flat search on the unindexed portion. LanceDB Cloud automates the optimize process, minimizing the impact on search speed.
--- a/docs/src/python/python.md
+++ b/docs/src/python/python.md
@@ -1,16 +1,6 @@
 # Python API Reference

-This section contains the API reference for the Python API. There is a
-synchronous and an asynchronous API client.
-
-The general flow of using the API is:
-
-1. Use [lancedb.connect][] or [lancedb.connect_async][] to connect to a database.
-2. Use the returned [lancedb.DBConnection][] or [lancedb.AsyncConnection][] to
-   create or open tables.
-3. Use the returned [lancedb.table.Table][] or [lancedb.AsyncTable][] to query
-   or modify tables.
-
+This section contains the API reference for the OSS Python API.

 ## Installation

--- a/docs/src/sql.md
+++ b/docs/src/sql.md
@@ -7,10 +7,6 @@ performed on the top-k results returned by the vector search. However, pre-filte
 option that performs the filter prior to vector search. This can be useful to narrow down on
 the search space on a very large dataset to reduce query latency.

-Note that both pre-filtering and post-filtering can yield false positives. For pre-filtering, if the filter is too selective, it might eliminate relevant items that the vector search would have otherwise identified as a good match. In this case, increasing `nprobes` parameter will help reduce such false positives. It is recommended to set `use_index=false` if you know that the filter is highly selective.
-
-Similarly, a highly selective post-filter can lead to false positives. Increasing both `nprobes` and `refine_factor` can mitigate this issue. When deciding between pre-filtering and post-filtering, pre-filtering is generally the safer choice if you're uncertain.
-
 <!-- Setup Code
 ```python
 import lancedb
@@ -61,9 +57,6 @@ const tbl = await db.createTable('myVectors', data)
        ```ts
        --8<-- "docs/src/sql_legacy.ts:search"
        ```
-!!! note
-
-    Creating a [scalar index](guides/scalar_index.md) accelerates filtering

 ## SQL filters

--- a/java/core/pom.xml
+++ b/java/core/pom.xml
@@ -8,7 +8,7 @@
    <parent>
        <groupId>com.lancedb</groupId>
        <artifactId>lancedb-parent</artifactId>
-        <version>0.14.0-beta.0</version>
+        <version>0.13.1-beta.0</version>
        <relativePath>../pom.xml</relativePath>
    </parent>

--- a/java/pom.xml
+++ b/java/pom.xml
@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.14.0-beta.0</version>
+    <version>0.13.1-beta.0</version>
    <packaging>pom</packaging>

    <name>LanceDB Parent</name>
--- a/node/package-lock.json
+++ b/node/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "vectordb",
-  "version": "0.14.0-beta.0",
+  "version": "0.13.1-beta.0",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "vectordb",
-      "version": "0.14.0-beta.0",
+      "version": "0.13.1-beta.0",
      "cpu": [
        "x64",
        "arm64"
@@ -52,14 +52,14 @@
        "uuid": "^9.0.0"
      },
      "optionalDependencies": {
-        "@lancedb/vectordb-darwin-arm64": "0.14.0-beta.0",
-        "@lancedb/vectordb-darwin-x64": "0.14.0-beta.0",
-        "@lancedb/vectordb-linux-arm64-gnu": "0.14.0-beta.0",
-        "@lancedb/vectordb-linux-arm64-musl": "0.14.0-beta.0",
-        "@lancedb/vectordb-linux-x64-gnu": "0.14.0-beta.0",
-        "@lancedb/vectordb-linux-x64-musl": "0.14.0-beta.0",
-        "@lancedb/vectordb-win32-arm64-msvc": "0.14.0-beta.0",
-        "@lancedb/vectordb-win32-x64-msvc": "0.14.0-beta.0"
+        "@lancedb/vectordb-darwin-arm64": "0.13.1-beta.0",
+        "@lancedb/vectordb-darwin-x64": "0.13.1-beta.0",
+        "@lancedb/vectordb-linux-arm64-gnu": "0.13.1-beta.0",
+        "@lancedb/vectordb-linux-arm64-musl": "0.13.1-beta.0",
+        "@lancedb/vectordb-linux-x64-gnu": "0.13.1-beta.0",
+        "@lancedb/vectordb-linux-x64-musl": "0.13.1-beta.0",
+        "@lancedb/vectordb-win32-arm64-msvc": "0.13.1-beta.0",
+        "@lancedb/vectordb-win32-x64-msvc": "0.13.1-beta.0"
      },
      "peerDependencies": {
        "@apache-arrow/ts": "^14.0.2",
--- a/node/package.json
+++ b/node/package.json
@@ -1,6 +1,6 @@
 {
  "name": "vectordb",
-  "version": "0.14.0-beta.0",
+  "version": "0.13.1-beta.0",
  "description": " Serverless, low-latency vector database for AI applications",
  "main": "dist/index.js",
  "types": "dist/index.d.ts",
@@ -91,13 +91,13 @@
    }
  },
  "optionalDependencies": {
-    "@lancedb/vectordb-darwin-x64": "0.14.0-beta.0",
-    "@lancedb/vectordb-darwin-arm64": "0.14.0-beta.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.14.0-beta.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.14.0-beta.0",
-    "@lancedb/vectordb-linux-x64-musl": "0.14.0-beta.0",
-    "@lancedb/vectordb-linux-arm64-musl": "0.14.0-beta.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.14.0-beta.0",
-    "@lancedb/vectordb-win32-arm64-msvc": "0.14.0-beta.0"
+    "@lancedb/vectordb-darwin-x64": "0.13.1-beta.0",
+    "@lancedb/vectordb-darwin-arm64": "0.13.1-beta.0",
+    "@lancedb/vectordb-linux-x64-gnu": "0.13.1-beta.0",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.13.1-beta.0",
+    "@lancedb/vectordb-linux-x64-musl": "0.13.1-beta.0",
+    "@lancedb/vectordb-linux-arm64-musl": "0.13.1-beta.0",
+    "@lancedb/vectordb-win32-x64-msvc": "0.13.1-beta.0",
+    "@lancedb/vectordb-win32-arm64-msvc": "0.13.1-beta.0"
  }
 }
--- a/nodejs/Cargo.toml
+++ b/nodejs/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.14.0-beta.0"
+version = "0.13.1-beta.0"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
--- a/nodejs/test/connection.test.ts
+++ b/nodejs/test/connection.test.ts
@@ -110,10 +110,7 @@ describe("given a connection", () => {
    let table = await db.createTable("test", data, { useLegacyFormat: true });

    const isV2 = async (table: Table) => {
-      const data = await table
-        .query()
-        .limit(10000)
-        .toArrow({ maxBatchLength: 100000 });
+      const data = await table.query().toArrow({ maxBatchLength: 100000 });
      console.log(data.batches.length);
      return data.batches.length < 5;
    };
--- a/nodejs/test/table.test.ts
+++ b/nodejs/test/table.test.ts
@@ -585,11 +585,11 @@ describe("When creating an index", () => {
    expect(fs.readdirSync(indexDir)).toHaveLength(1);

    for await (const r of tbl.query().where("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(10);
+      expect(r.numRows).toBe(298);
    }
    // should also work with 'filter' alias
    for await (const r of tbl.query().filter("id > 1").select(["id"])) {
-      expect(r.numRows).toBe(10);
+      expect(r.numRows).toBe(298);
    }
  });

--- a/nodejs/npm/darwin-arm64/package.json
+++ b/nodejs/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
--- a/nodejs/npm/darwin-x64/package.json
+++ b/nodejs/npm/darwin-x64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",
--- a/nodejs/npm/linux-arm64-gnu/package.json
+++ b/nodejs/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
--- a/nodejs/npm/linux-arm64-musl/package.json
+++ b/nodejs/npm/linux-arm64-musl/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
--- a/nodejs/npm/linux-x64-gnu/package.json
+++ b/nodejs/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
--- a/nodejs/npm/linux-x64-musl/package.json
+++ b/nodejs/npm/linux-x64-musl/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
--- a/nodejs/npm/win32-arm64-msvc/package.json
+++ b/nodejs/npm/win32-arm64-msvc/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.14.0-beta.0",
+  "version": "0.13.1-beta.0",
  "os": [
    "win32"
  ],
--- a/nodejs/npm/win32-x64-msvc/package.json
+++ b/nodejs/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.14.0-beta.0",
+	"version": "0.13.1-beta.0",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -10,7 +10,7 @@
    "vector database",
    "ann"
  ],
-  "version": "0.14.0-beta.0",
+  "version": "0.13.1-beta.0",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
--- a/python/.bumpversion.toml
+++ b/python/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.17.0-beta.1"
+current_version = "0.16.1-beta.0"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.17.0-beta.1"
+version = "0.16.1-beta.0"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -3,7 +3,8 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
    "deprecation",
-    "pylance==0.20.0b3",
+    "nest-asyncio~=1.0",
+    "pylance==0.20.0b2",
    "tqdm>=4.27.0",
    "pydantic>=1.10",
    "packaging",
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -1502,11 +1502,10 @@ class AsyncQueryBase(object):
        ...     print(plan)
        >>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
        ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
-          GlobalLimitExec: skip=0, fetch=10
-            FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
-                KNNVectorDistance: metric=l2
-                  LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false
+          FilterExec: _distance@2 IS NOT NULL
+            SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST], preserve_partitioning=[false]
+              KNNVectorDistance: metric=l2
+                LanceScan: uri=..., projection=[vector], row_id=true, row_addr=false, ordered=false

        Parameters
        ----------
--- a/python/python/lancedb/remote/background_loop.py
+++ b/python/python/lancedb/remote/background_loop.py
@@ -1,25 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-import asyncio
-import threading
-
-
-class BackgroundEventLoop:
-    """
-    A background event loop that can run futures.
-
-    Used to bridge sync and async code, without messing with users event loops.
-    """
-
-    def __init__(self):
-        self.loop = asyncio.new_event_loop()
-        self.thread = threading.Thread(
-            target=self.loop.run_forever,
-            name="LanceDBBackgroundEventLoop",
-            daemon=True,
-        )
-        self.thread.start()
-
-    def run(self, future):
-        return asyncio.run_coroutine_threadsafe(future, self.loop).result()
--- a/python/python/lancedb/remote/db.py
+++ b/python/python/lancedb/remote/db.py
@@ -11,6 +11,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.

+import asyncio
 from datetime import timedelta
 import logging
 from concurrent.futures import ThreadPoolExecutor
@@ -20,7 +21,6 @@ import warnings

 from lancedb import connect_async
 from lancedb.remote import ClientConfig
-from lancedb.remote.background_loop import BackgroundEventLoop
 import pyarrow as pa
 from overrides import override

@@ -31,8 +31,6 @@ from ..pydantic import LanceModel
 from ..table import Table
 from ..util import validate_table_name

-LOOP = BackgroundEventLoop()
-

 class RemoteDBConnection(DBConnection):
    """A connection to a remote LanceDB database."""
@@ -88,9 +86,18 @@ class RemoteDBConnection(DBConnection):
            raise ValueError(f"Invalid scheme: {parsed.scheme}, only accepts db://")
        self.db_name = parsed.netloc

+        import nest_asyncio
+
+        nest_asyncio.apply()
+        try:
+            self._loop = asyncio.get_running_loop()
+        except RuntimeError:
+            self._loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._loop)
+
        self.client_config = client_config

-        self._conn = LOOP.run(
+        self._conn = self._loop.run_until_complete(
            connect_async(
                db_url,
                api_key=api_key,
@@ -120,7 +127,9 @@ class RemoteDBConnection(DBConnection):
        -------
        An iterator of table names.
        """
-        return LOOP.run(self._conn.table_names(start_after=page_token, limit=limit))
+        return self._loop.run_until_complete(
+            self._conn.table_names(start_after=page_token, limit=limit)
+        )

    @override
    def open_table(self, name: str, *, index_cache_size: Optional[int] = None) -> Table:
@@ -143,8 +152,8 @@ class RemoteDBConnection(DBConnection):
                " (there is no local cache to configure)"
            )

-        table = LOOP.run(self._conn.open_table(name))
-        return RemoteTable(table, self.db_name)
+        table = self._loop.run_until_complete(self._conn.open_table(name))
+        return RemoteTable(table, self.db_name, self._loop)

    @override
    def create_table(
@@ -259,7 +268,7 @@ class RemoteDBConnection(DBConnection):

        from .table import RemoteTable

-        table = LOOP.run(
+        table = self._loop.run_until_complete(
            self._conn.create_table(
                name,
                data,
@@ -269,7 +278,7 @@ class RemoteDBConnection(DBConnection):
                fill_value=fill_value,
            )
        )
-        return RemoteTable(table, self.db_name)
+        return RemoteTable(table, self.db_name, self._loop)

    @override
    def drop_table(self, name: str):
@@ -280,7 +289,7 @@ class RemoteDBConnection(DBConnection):
        name: str
            The name of the table.
        """
-        LOOP.run(self._conn.drop_table(name))
+        self._loop.run_until_complete(self._conn.drop_table(name))

    @override
    def rename_table(self, cur_name: str, new_name: str):
@@ -293,7 +302,7 @@ class RemoteDBConnection(DBConnection):
        new_name: str
            The new name of the table.
        """
-        LOOP.run(self._conn.rename_table(cur_name, new_name))
+        self._loop.run_until_complete(self._conn.rename_table(cur_name, new_name))

    async def close(self):
        """Close the connection to the database."""
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -12,12 +12,12 @@
 #  limitations under the License.

 from datetime import timedelta
+import asyncio
 import logging
 from functools import cached_property
 from typing import Dict, Iterable, List, Optional, Union, Literal

 from lancedb.index import FTS, BTree, Bitmap, HnswPq, HnswSq, IvfPq, LabelList
-from lancedb.remote.db import LOOP
 import pyarrow as pa

 from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
@@ -33,7 +33,9 @@ class RemoteTable(Table):
        self,
        table: AsyncTable,
        db_name: str,
+        loop: Optional[asyncio.AbstractEventLoop] = None,
    ):
+        self._loop = loop
        self._table = table
        self.db_name = db_name

@@ -54,12 +56,12 @@ class RemoteTable(Table):
        of this Table

        """
-        return LOOP.run(self._table.schema())
+        return self._loop.run_until_complete(self._table.schema())

    @property
    def version(self) -> int:
        """Get the current version of the table"""
-        return LOOP.run(self._table.version())
+        return self._loop.run_until_complete(self._table.version())

    @cached_property
    def embedding_functions(self) -> dict:
@@ -96,11 +98,11 @@ class RemoteTable(Table):

    def list_indices(self):
        """List all the indices on the table"""
-        return LOOP.run(self._table.list_indices())
+        return self._loop.run_until_complete(self._table.list_indices())

    def index_stats(self, index_uuid: str):
        """List all the stats of a specified index"""
-        return LOOP.run(self._table.index_stats(index_uuid))
+        return self._loop.run_until_complete(self._table.index_stats(index_uuid))

    def create_scalar_index(
        self,
@@ -130,7 +132,9 @@ class RemoteTable(Table):
        else:
            raise ValueError(f"Unknown index type: {index_type}")

-        LOOP.run(self._table.create_index(column, config=config, replace=replace))
+        self._loop.run_until_complete(
+            self._table.create_index(column, config=config, replace=replace)
+        )

    def create_fts_index(
        self,
@@ -140,7 +144,9 @@ class RemoteTable(Table):
        with_position: bool = True,
    ):
        config = FTS(with_position=with_position)
-        LOOP.run(self._table.create_index(column, config=config, replace=replace))
+        self._loop.run_until_complete(
+            self._table.create_index(column, config=config, replace=replace)
+        )

    def create_index(
        self,
@@ -221,7 +227,9 @@ class RemoteTable(Table):
                " 'IVF_PQ', 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
            )

-        LOOP.run(self._table.create_index(vector_column_name, config=config))
+        self._loop.run_until_complete(
+            self._table.create_index(vector_column_name, config=config)
+        )

    def add(
        self,
@@ -253,7 +261,7 @@ class RemoteTable(Table):
            The value to use when filling vectors. Only used if on_bad_vectors="fill".

        """
-        LOOP.run(
+        self._loop.run_until_complete(
            self._table.add(
                data, mode=mode, on_bad_vectors=on_bad_vectors, fill_value=fill_value
            )
@@ -341,7 +349,9 @@ class RemoteTable(Table):
    def _execute_query(
        self, query: Query, batch_size: Optional[int] = None
    ) -> pa.RecordBatchReader:
-        return LOOP.run(self._table._execute_query(query, batch_size=batch_size))
+        return self._loop.run_until_complete(
+            self._table._execute_query(query, batch_size=batch_size)
+        )

    def merge_insert(self, on: Union[str, Iterable[str]]) -> LanceMergeInsertBuilder:
        """Returns a [`LanceMergeInsertBuilder`][lancedb.merge.LanceMergeInsertBuilder]
@@ -358,7 +368,9 @@ class RemoteTable(Table):
        on_bad_vectors: str,
        fill_value: float,
    ):
-        LOOP.run(self._table._do_merge(merge, new_data, on_bad_vectors, fill_value))
+        self._loop.run_until_complete(
+            self._table._do_merge(merge, new_data, on_bad_vectors, fill_value)
+        )

    def delete(self, predicate: str):
        """Delete rows from the table.
@@ -407,7 +419,7 @@ class RemoteTable(Table):
           x      vector  _distance # doctest: +SKIP
        0  2  [3.0, 4.0]       85.0 # doctest: +SKIP
        """
-        LOOP.run(self._table.delete(predicate))
+        self._loop.run_until_complete(self._table.delete(predicate))

    def update(
        self,
@@ -457,7 +469,7 @@ class RemoteTable(Table):
        2  2  [10.0, 10.0] # doctest: +SKIP

        """
-        LOOP.run(
+        self._loop.run_until_complete(
            self._table.update(where=where, updates=values, updates_sql=values_sql)
        )

@@ -487,7 +499,7 @@ class RemoteTable(Table):
        )

    def count_rows(self, filter: Optional[str] = None) -> int:
-        return LOOP.run(self._table.count_rows(filter))
+        return self._loop.run_until_complete(self._table.count_rows(filter))

    def add_columns(self, transforms: Dict[str, str]):
        raise NotImplementedError(
--- a/python/python/lancedb/util.py
+++ b/python/python/lancedb/util.py
@@ -11,6 +11,8 @@ from datetime import date, datetime
 from functools import singledispatch
 from typing import Tuple, Union, Optional, Any
 from urllib.parse import urlparse
+from threading import Lock
+from contextlib import contextmanager

 import numpy as np
 import pyarrow as pa
@@ -314,3 +316,27 @@ def deprecated(func):
 def validate_table_name(name: str):
    """Verify the table name is valid."""
    native_validate_table_name(name)
+
+
+class ConnectionPool:
+    def __init__(self, connection_factory, *, max_size: Optional[int] = None):
+        self.max_size = max_size
+        self._connection_factory = connection_factory
+        self._pool = []
+        self._lock = Lock()
+
+    @contextmanager
+    def connection(self):
+        with self._lock:
+            if self._pool:
+                conn = self._pool.pop()
+            else:
+                conn = self._connection_factory()
+
+        # release the lock before yielding
+        try:
+            yield conn
+        finally:
+            with self._lock:
+                if self.max_size is None or len(self._pool) < self.max_size:
+                    self._pool.append(conn)
--- a/python/python/tests/test_db.py
+++ b/python/python/tests/test_db.py
@@ -599,9 +599,7 @@ async def test_create_in_v2_mode(tmp_path):
    )

    async def is_in_v2_mode(tbl):
-        batches = (
-            await tbl.query().limit(10 * 1024).to_batches(max_batch_length=1024 * 10)
-        )
+        batches = await tbl.query().to_batches(max_batch_length=1024 * 10)
        num_batches = 0
        async for batch in batches:
            num_batches += 1
--- a/python/python/tests/test_remote_db.py
+++ b/python/python/tests/test_remote_db.py
@@ -1,19 +1,21 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-from concurrent.futures import ThreadPoolExecutor
 import contextlib
 from datetime import timedelta
 import http.server
 import json
 import threading
+from concurrent.futures import ThreadPoolExecutor
 from unittest.mock import MagicMock
 import uuid

 import lancedb
 from lancedb.conftest import MockTextEmbeddingFunction
 from lancedb.remote import ClientConfig
+from lancedb.util import ConnectionPool
 from lancedb.remote.errors import HttpError, RetryError
+import lancedb.util
 import pytest
 import pyarrow as pa

@@ -56,6 +58,34 @@ def mock_lancedb_connection(handler):
            handle.join()


+@contextlib.contextmanager
+def mock_lancedb_connection_pool(handler):
+    with http.server.HTTPServer(
+        ("localhost", 8080), make_mock_http_handler(handler)
+    ) as server:
+        handle = threading.Thread(target=server.serve_forever)
+        handle.start()
+
+        def conn_factory():
+            lancedb.connect(
+                "db://dev",
+                api_key="fake",
+                host_override="http://localhost:8080",
+                client_config={
+                    "retry_config": {"retries": 2},
+                    "timeout_config": {
+                        "connect_timeout": 1,
+                    },
+                },
+            )
+
+        try:
+            yield ConnectionPool(conn_factory)
+        finally:
+            server.shutdown()
+            handle.join()
+
+
@contextlib.asynccontextmanager
 async def mock_lancedb_connection_async(handler):
    with http.server.HTTPServer(
@@ -188,49 +218,7 @@ async def test_retry_error():
        assert cause.status_code == 429


-def test_table_add_in_threadpool():
-    def handler(request):
-        if request.path == "/v1/table/test/insert/":
-            request.send_response(200)
-            request.end_headers()
-        elif request.path == "/v1/table/test/create/":
-            request.send_response(200)
-            request.send_header("Content-Type", "application/json")
-            request.end_headers()
-            request.wfile.write(b"{}")
-        elif request.path == "/v1/table/test/describe/":
-            request.send_response(200)
-            request.send_header("Content-Type", "application/json")
-            request.end_headers()
-            payload = json.dumps(
-                dict(
-                    version=1,
-                    schema=dict(
-                        fields=[
-                            dict(name="id", type={"type": "int64"}, nullable=False),
-                        ]
-                    ),
-                )
-            )
-            request.wfile.write(payload.encode())
-        else:
-            request.send_response(404)
-            request.end_headers()
-
-    with mock_lancedb_connection(handler) as db:
-        table = db.create_table("test", [{"id": 1}])
-        with ThreadPoolExecutor(3) as executor:
-            futures = []
-            for _ in range(10):
-                future = executor.submit(table.add, [{"id": 1}])
-                futures.append(future)
-
-            for future in futures:
-                future.result()
-
-
-@contextlib.contextmanager
-def query_test_table(query_handler):
+def http_handler(query_handler):
    def handler(request):
        if request.path == "/v1/table/test/describe/":
            request.send_response(200)
@@ -254,7 +242,12 @@ def query_test_table(query_handler):
            request.send_response(404)
            request.end_headers()

-    with mock_lancedb_connection(handler) as db:
+    return handler
+
+
+@contextlib.contextmanager
+def query_test_table(connection_ctx_mgr):
+    with connection_ctx_mgr as db:
        assert repr(db) == "RemoteConnect(name=dev)"
        table = db.open_table("test")
        assert repr(table) == "RemoteTable(dev.test)"
@@ -262,6 +255,7 @@ def query_test_table(query_handler):


 def test_query_sync_minimal():
+    @http_handler
    def handler(body):
        assert body == {
            "distance_type": "l2",
@@ -276,13 +270,53 @@ def test_query_sync_minimal():

        return pa.table({"id": [1, 2, 3]})

-    with query_test_table(handler) as table:
+    with query_test_table(mock_lancedb_connection(handler)) as table:
+        data = table.search([1, 2, 3]).to_list()
+        expected = [{"id": 1}, {"id": 2}, {"id": 3}]
+        assert data == expected
+
+    with query_test_table(mock_lancedb_connection_pool(handler).connection()) as table:
        data = table.search([1, 2, 3]).to_list()
        expected = [{"id": 1}, {"id": 2}, {"id": 3}]
        assert data == expected


+def test_query_sync_minimal_threaded():
+    num_query = 0
+
+    @http_handler
+    def handler(body):
+        assert body == {
+            "distance_type": "l2",
+            "k": 10,
+            "prefilter": False,
+            "refine_factor": None,
+            "ef": None,
+            "vector": [1.0, 2.0, 3.0],
+            "nprobes": 20,
+            "version": None,
+        }
+        nonlocal num_query
+        num_query += 1
+
+        return pa.table({"id": [1, 2, 3]})
+
+    pool = mock_lancedb_connection_pool(handler)
+
+    def _query(i):
+        with query_test_table(pool.connection()) as table:
+            data = table.search([1, 2, 3]).to_list()
+            expected = [{"id": 1}, {"id": 2}, {"id": 3}]
+            assert data == expected
+
+    with ThreadPoolExecutor as exec:
+        exec.map(_query, range(1000))
+
+    assert num_query == 1000
+
+
 def test_query_sync_empty_query():
+    @http_handler
    def handler(body):
        assert body == {
            "k": 10,
@@ -294,7 +328,12 @@ def test_query_sync_empty_query():

        return pa.table({"id": [1, 2, 3]})

-    with query_test_table(handler) as table:
+    with query_test_table(mock_lancedb_connection(handler)) as table:
+        data = table.search(None).where("true").select(["id"]).limit(10).to_list()
+        expected = [{"id": 1}, {"id": 2}, {"id": 3}]
+        assert data == expected
+
+    with query_test_table(mock_lancedb_connection_pool(handler).connection()) as table:
        data = table.search(None).where("true").select(["id"]).limit(10).to_list()
        expected = [{"id": 1}, {"id": 2}, {"id": 3}]
        assert data == expected
--- a/rust/ffi/node/Cargo.toml
+++ b/rust/ffi/node/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-node"
-version = "0.14.0-beta.0"
+version = "0.13.1-beta.0"
 description = "Serverless, low-latency vector database for AI applications"
 license.workspace = true
 edition.workspace = true
--- a/rust/lancedb/Cargo.toml
+++ b/rust/lancedb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.14.0-beta.0"
+version = "0.13.1-beta.0"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -27,7 +27,6 @@ half = { workspace = true }
 lazy_static.workspace = true
 lance = { workspace = true }
 lance-datafusion.workspace = true
-lance-io = { workspace = true }
 lance-index = { workspace = true }
 lance-table = { workspace = true }
 lance-linalg = { workspace = true }
--- a/rust/lancedb/src/connection.rs
+++ b/rust/lancedb/src/connection.rs
@@ -38,9 +38,6 @@ use crate::table::{NativeTable, TableDefinition, WriteOptions};
 use crate::utils::validate_table_name;
 use crate::Table;
 pub use lance_encoding::version::LanceFileVersion;
-#[cfg(feature = "remote")]
-use lance_io::object_store::StorageOptions;
-use lance_table::io::commit::commit_handler_from_url;

 pub const LANCE_FILE_EXTENSION: &str = "lance";

@@ -136,7 +133,7 @@ impl IntoArrow for NoData {

 /// A builder for configuring a [`Connection::create_table`] operation
 pub struct CreateTableBuilder<const HAS_DATA: bool, T: IntoArrow> {
-    pub(crate) parent: Arc<dyn ConnectionInternal>,
+    parent: Arc<dyn ConnectionInternal>,
    pub(crate) name: String,
    pub(crate) data: Option<T>,
    pub(crate) mode: CreateTableMode,
@@ -344,7 +341,7 @@ pub struct OpenTableBuilder {
 }

 impl OpenTableBuilder {
-    pub(crate) fn new(parent: Arc<dyn ConnectionInternal>, name: String) -> Self {
+    fn new(parent: Arc<dyn ConnectionInternal>, name: String) -> Self {
        Self {
            parent,
            name,
@@ -720,14 +717,12 @@ impl ConnectBuilder {
            message: "An api_key is required when connecting to LanceDb Cloud".to_string(),
        })?;

-        let storage_options = StorageOptions(self.storage_options.clone());
        let internal = Arc::new(crate::remote::db::RemoteDatabase::try_new(
            &self.uri,
            &api_key,
            &region,
            self.host_override,
            self.client_config,
-            storage_options.into(),
        )?);
        Ok(Connection {
            internal,
@@ -860,7 +855,7 @@ impl Database {
                let table_base_uri = if let Some(store) = engine {
                    static WARN_ONCE: std::sync::Once = std::sync::Once::new();
                    WARN_ONCE.call_once(|| {
-                        log::warn!("Specifying engine is not a publicly supported feature in lancedb yet. THE API WILL CHANGE");
+                        log::warn!("Specifing engine is not a publicly supported feature in lancedb yet. THE API WILL CHANGE");
                    });
                    let old_scheme = url.scheme().to_string();
                    let new_scheme = format!("{}+{}", old_scheme, store);
@@ -1041,7 +1036,6 @@ impl ConnectionInternal for Database {
        };

        let mut write_params = options.write_options.lance_write_params.unwrap_or_default();
-
        if matches!(&options.mode, CreateTableMode::Overwrite) {
            write_params.mode = WriteMode::Overwrite;
        }
@@ -1128,7 +1122,7 @@ impl ConnectionInternal for Database {
        let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
        let full_path = self.base_path.child(dir_name.clone());
        self.object_store
-            .remove_dir_all(full_path.clone())
+            .remove_dir_all(full_path)
            .await
            .map_err(|err| match err {
                // this error is not lance::Error::DatasetNotFound,
@@ -1138,19 +1132,6 @@ impl ConnectionInternal for Database {
                },
                _ => Error::from(err),
            })?;
-
-        let object_store_params = ObjectStoreParams {
-            storage_options: Some(self.storage_options.clone()),
-            ..Default::default()
-        };
-        let mut uri = self.uri.clone();
-        if let Some(query_string) = &self.query_string {
-            uri.push_str(&format!("?{}", query_string));
-        }
-        let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params))
-            .await
-            .unwrap();
-        commit_handler.delete(&full_path).await.unwrap();
        Ok(())
    }

@@ -1188,7 +1169,6 @@ mod tests {
    use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
    use tempfile::tempdir;

-    use crate::query::QueryBase;
    use crate::query::{ExecutableQuery, QueryExecutionOptions};

    use super::*;
@@ -1316,7 +1296,6 @@ mod tests {
        // In v1 the row group size will trump max_batch_length
        let batches = tbl
            .query()
-            .limit(20000)
            .execute_with_options(QueryExecutionOptions {
                max_batch_length: 50000,
                ..Default::default()
--- a/rust/lancedb/src/query.rs
+++ b/rust/lancedb/src/query.rs
@@ -596,7 +596,7 @@ impl Query {
    pub(crate) fn new(parent: Arc<dyn TableInternal>) -> Self {
        Self {
            parent,
-            limit: Some(DEFAULT_TOP_K),
+            limit: None,
            offset: None,
            filter: None,
            full_text_search: None,
--- a/rust/lancedb/src/remote/client.rs
+++ b/rust/lancedb/src/remote/client.rs
@@ -21,7 +21,6 @@ use reqwest::{
 };

 use crate::error::{Error, Result};
-use crate::remote::db::RemoteOptions;

 const REQUEST_ID_HEADER: &str = "x-request-id";

@@ -216,7 +215,6 @@ impl RestfulLanceDbClient<Sender> {
        region: &str,
        host_override: Option<String>,
        client_config: ClientConfig,
-        options: &RemoteOptions,
    ) -> Result<Self> {
        let parsed_url = url::Url::parse(db_url).map_err(|err| Error::InvalidInput {
            message: format!("db_url is not a valid URL. '{db_url}'. Error: {err}"),
@@ -257,7 +255,6 @@ impl RestfulLanceDbClient<Sender> {
                region,
                db_name,
                host_override.is_some(),
-                options,
            )?)
            .user_agent(client_config.user_agent)
            .build()
@@ -265,7 +262,6 @@ impl RestfulLanceDbClient<Sender> {
                message: "Failed to build HTTP client".into(),
                source: Some(Box::new(err)),
            })?;
-
        let host = match host_override {
            Some(host_override) => host_override,
            None => format!("https://{}.{}.api.lancedb.com", db_name, region),
@@ -291,7 +287,6 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
        region: &str,
        db_name: &str,
        has_host_override: bool,
-        options: &RemoteOptions,
    ) -> Result<HeaderMap> {
        let mut headers = HeaderMap::new();
        headers.insert(
@@ -318,23 +313,6 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
            );
        }

-        if let Some(v) = options.0.get("account_name") {
-            headers.insert(
-                "x-azure-storage-account-name",
-                HeaderValue::from_str(v).map_err(|_| Error::InvalidInput {
-                    message: format!("non-ascii storage account name '{}' provided", db_name),
-                })?,
-            );
-        }
-        if let Some(v) = options.0.get("azure_storage_account_name") {
-            headers.insert(
-                "x-azure-storage-account-name",
-                HeaderValue::from_str(v).map_err(|_| Error::InvalidInput {
-                    message: format!("non-ascii storage account name '{}' provided", db_name),
-                })?,
-            );
-        }
-
        Ok(headers)
    }

--- a/rust/lancedb/src/remote/db.rs
+++ b/rust/lancedb/src/remote/db.rs
@@ -12,21 +12,18 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.

-use std::collections::HashMap;
 use std::sync::Arc;

 use arrow_array::RecordBatchReader;
 use async_trait::async_trait;
 use http::StatusCode;
-use lance_io::object_store::StorageOptions;
 use moka::future::Cache;
 use reqwest::header::CONTENT_TYPE;
 use serde::Deserialize;
 use tokio::task::spawn_blocking;

 use crate::connection::{
-    ConnectionInternal, CreateTableBuilder, CreateTableMode, NoData, OpenTableBuilder,
-    TableNamesBuilder,
+    ConnectionInternal, CreateTableBuilder, NoData, OpenTableBuilder, TableNamesBuilder,
 };
 use crate::embeddings::EmbeddingRegistry;
 use crate::error::Result;
@@ -55,16 +52,9 @@ impl RemoteDatabase {
        region: &str,
        host_override: Option<String>,
        client_config: ClientConfig,
-        options: RemoteOptions,
    ) -> Result<Self> {
-        let client = RestfulLanceDbClient::try_new(
-            uri,
-            api_key,
-            region,
-            host_override,
-            client_config,
-            &options,
-        )?;
+        let client =
+            RestfulLanceDbClient::try_new(uri, api_key, region, host_override, client_config)?;

        let table_cache = Cache::builder()
            .time_to_live(std::time::Duration::from_secs(300))
@@ -105,16 +95,6 @@ impl<S: HttpSend> std::fmt::Display for RemoteDatabase<S> {
    }
 }

-impl From<&CreateTableMode> for &'static str {
-    fn from(val: &CreateTableMode) -> Self {
-        match val {
-            CreateTableMode::Create => "create",
-            CreateTableMode::Overwrite => "overwrite",
-            CreateTableMode::ExistOk(_) => "exist_ok",
-        }
-    }
-}
-
 #[async_trait]
 impl<S: HttpSend> ConnectionInternal for RemoteDatabase<S> {
    async fn table_names(&self, options: TableNamesBuilder) -> Result<Vec<String>> {
@@ -153,40 +133,14 @@ impl<S: HttpSend> ConnectionInternal for RemoteDatabase<S> {
        let req = self
            .client
            .post(&format!("/v1/table/{}/create/", options.name))
-            .query(&[("mode", Into::<&str>::into(&options.mode))])
            .body(data_buffer)
            .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE);
-
        let (request_id, rsp) = self.client.send(req, false).await?;

        if rsp.status() == StatusCode::BAD_REQUEST {
            let body = rsp.text().await.err_to_http(request_id.clone())?;
            if body.contains("already exists") {
-                return match options.mode {
-                    CreateTableMode::Create => {
-                        Err(crate::Error::TableAlreadyExists { name: options.name })
-                    }
-                    CreateTableMode::ExistOk(callback) => {
-                        let builder = OpenTableBuilder::new(options.parent, options.name);
-                        let builder = (callback)(builder);
-                        builder.execute().await
-                    }
-
-                    // This should not happen, as we explicitly set the mode to overwrite and the server
-                    // shouldn't return an error if the table already exists.
-                    //
-                    // However if the server is an older version that doesn't support the mode parameter,
-                    // then we'll get the 400 response.
-                    CreateTableMode::Overwrite => Err(crate::Error::Http {
-                        source: format!(
-                            "unexpected response from server for create mode overwrite: {}",
-                            body
-                        )
-                        .into(),
-                        request_id,
-                        status_code: Some(StatusCode::BAD_REQUEST),
-                    }),
-                };
+                return Err(crate::Error::TableAlreadyExists { name: options.name });
            } else {
                return Err(crate::Error::InvalidInput { message: body });
            }
@@ -252,29 +206,6 @@ impl<S: HttpSend> ConnectionInternal for RemoteDatabase<S> {
    }
 }

-/// RemoteOptions contains a subset of StorageOptions that are compatible with Remote LanceDB connections
-#[derive(Clone, Debug, Default)]
-pub struct RemoteOptions(pub HashMap<String, String>);
-
-impl RemoteOptions {
-    pub fn new(options: HashMap<String, String>) -> Self {
-        Self(options)
-    }
-}
-
-impl From<StorageOptions> for RemoteOptions {
-    fn from(options: StorageOptions) -> Self {
-        let supported_opts = vec!["account_name", "azure_storage_account_name"];
-        let mut filtered = HashMap::new();
-        for opt in supported_opts {
-            if let Some(v) = options.0.get(opt) {
-                filtered.insert(opt.to_string(), v.to_string());
-            }
-        }
-        RemoteOptions::new(filtered)
-    }
-}
-
 #[cfg(test)]
 mod tests {
    use std::sync::{Arc, OnceLock};
@@ -282,9 +213,7 @@ mod tests {
    use arrow_array::{Int32Array, RecordBatch, RecordBatchIterator};
    use arrow_schema::{DataType, Field, Schema};

-    use crate::connection::ConnectBuilder;
    use crate::{
-        connection::CreateTableMode,
        remote::{ARROW_STREAM_CONTENT_TYPE, JSON_CONTENT_TYPE},
        Connection, Error,
    };
@@ -453,73 +382,6 @@ mod tests {
        );
    }

-    #[tokio::test]
-    async fn test_create_table_modes() {
-        let test_cases = [
-            (None, "mode=create"),
-            (Some(CreateTableMode::Create), "mode=create"),
-            (Some(CreateTableMode::Overwrite), "mode=overwrite"),
-            (
-                Some(CreateTableMode::ExistOk(Box::new(|b| b))),
-                "mode=exist_ok",
-            ),
-        ];
-
-        for (mode, expected_query_string) in test_cases {
-            let conn = Connection::new_with_handler(move |request| {
-                assert_eq!(request.method(), &reqwest::Method::POST);
-                assert_eq!(request.url().path(), "/v1/table/table1/create/");
-                assert_eq!(request.url().query(), Some(expected_query_string));
-
-                http::Response::builder().status(200).body("").unwrap()
-            });
-
-            let data = RecordBatch::try_new(
-                Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
-                vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-            )
-            .unwrap();
-            let reader = RecordBatchIterator::new([Ok(data.clone())], data.schema());
-            let mut builder = conn.create_table("table1", reader);
-            if let Some(mode) = mode {
-                builder = builder.mode(mode);
-            }
-            builder.execute().await.unwrap();
-        }
-
-        // check that the open table callback is called with exist_ok
-        let conn = Connection::new_with_handler(|request| match request.url().path() {
-            "/v1/table/table1/create/" => http::Response::builder()
-                .status(400)
-                .body("Table table1 already exists")
-                .unwrap(),
-            "/v1/table/table1/describe/" => http::Response::builder().status(200).body("").unwrap(),
-            _ => {
-                panic!("unexpected path: {:?}", request.url().path());
-            }
-        });
-        let data = RecordBatch::try_new(
-            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
-            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
-        )
-        .unwrap();
-
-        let called: Arc<OnceLock<bool>> = Arc::new(OnceLock::new());
-        let reader = RecordBatchIterator::new([Ok(data.clone())], data.schema());
-        let called_in_cb = called.clone();
-        conn.create_table("table1", reader)
-            .mode(CreateTableMode::ExistOk(Box::new(move |b| {
-                called_in_cb.clone().set(true).unwrap();
-                b
-            })))
-            .execute()
-            .await
-            .unwrap();
-
-        let called = *called.get().unwrap_or(&false);
-        assert!(called);
-    }
-
    #[tokio::test]
    async fn test_create_table_empty() {
        let conn = Connection::new_with_handler(|request| {
@@ -574,16 +436,4 @@ mod tests {
        });
        conn.rename_table("table1", "table2").await.unwrap();
    }
-
-    #[tokio::test]
-    async fn test_connect_remote_options() {
-        let db_uri = "db://my-container/my-prefix";
-        let _ = ConnectBuilder::new(db_uri)
-            .region("us-east-1")
-            .api_key("my-api-key")
-            .storage_options(vec![("azure_storage_account_name", "my-storage-account")])
-            .execute()
-            .await
-            .unwrap();
-    }
 }
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -1227,7 +1227,6 @@ mod tests {
                "prefilter": true,
                "distance_type": "l2",
                "nprobes": 20,
-                "k": 10,
                "ef": Option::<usize>::None,
                "refine_factor": null,
                "version": null,