Revert "fix: pass storage_options in LanceNamespaceDatabase::create_table"

This reverts commit 1078983993.

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.23.0"
+current_version = "0.24.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

.github/workflows/codex-update-lance-dependency.yml

@@ -75,6 +75,13 @@ jobs:
           VERSION="${VERSION#v}"
           BRANCH_NAME="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"
 
+          # Use "chore" for beta/rc versions, "feat" for stable releases
+          if [[ "${VERSION}" == *beta* ]] || [[ "${VERSION}" == *rc* ]]; then
+            COMMIT_TYPE="chore"
+          else
+            COMMIT_TYPE="feat"
+          fi
+
           cat <<EOF >/tmp/codex-prompt.txt
           You are running inside the lancedb repository on a GitHub Actions runner. Update the Lance dependency to version ${VERSION} and prepare a pull request for maintainers to review.
 
@@ -84,10 +91,10 @@ jobs:
           3. After clippy succeeds, run "cargo fmt --all" to format the workspace.
           4. Ensure the repository is clean except for intentional changes. Inspect "git status --short" and "git diff" to confirm the dependency update and any required fixes.
           5. Create and switch to a new branch named "${BRANCH_NAME}" (replace any duplicated hyphens if necessary).
-          6. Stage all relevant files with "git add -A". Commit using the message "chore: update lance dependency to v${VERSION}".
+          6. Stage all relevant files with "git add -A". Commit using the message "${COMMIT_TYPE}: update lance dependency to v${VERSION}".
           7. Push the branch to origin. If the branch already exists, force-push your changes.
           8. env "GH_TOKEN" is available, use "gh" tools for github related operations like creating pull request.
-          9. Create a pull request targeting "main" with title "chore: update lance dependency to v${VERSION}". In the body, summarize the dependency bump, clippy/fmt verification, and link the triggering tag (${TAG}).
+          9. Create a pull request targeting "main" with title "${COMMIT_TYPE}: update lance dependency to v${VERSION}". First, write the PR body to /tmp/pr-body.md using a heredoc (cat <<'EOF' > /tmp/pr-body.md). The body should summarize the dependency bump, clippy/fmt verification, and link the triggering tag (${TAG}). Then run "gh pr create --body-file /tmp/pr-body.md".
           10. After creating the PR, display the PR URL, "git status --short", and a concise summary of the commands run and their results.
 
           Constraints:

.github/workflows/rust.yml

@@ -48,6 +48,8 @@ jobs:
         run: cargo fmt --all -- --check
       - name: Run clippy
         run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
+      - name: Run clippy (without remote feature)
+        run: cargo clippy --profile ci --workspace --tests -- -D warnings
 
   build-no-lock:
     runs-on: ubuntu-24.04
@@ -167,13 +169,13 @@ jobs:
       - name: Build
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo build --profile ci --features remote --tests --locked --target ${{ matrix.target }}
+          cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }}
       - name: Run tests
         # Can only run tests when target matches host
         if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
         run: |
           $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
-          cargo test --profile ci --features remote --locked
+          cargo test --profile ci --features aws,remote --locked
 
   msrv:
     # Check the minimum supported Rust version
@@ -181,7 +183,7 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        msrv: ["1.78.0"] # This should match up with rust-version in Cargo.toml
+        msrv: ["1.88.0"] # This should match up with rust-version in Cargo.toml
     env:
       # Need up-to-date compilers for kernels
       CC: clang-18
@@ -212,4 +214,6 @@ jobs:
           cargo update -p aws-sdk-sts --precise 1.51.0
           cargo update -p home --precise 0.5.9
       - name: cargo +${{ matrix.msrv }} check
+        env:
+          RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
         run: cargo check --profile ci --workspace --tests --benches --all-features

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "adler2"
@@ -3141,8 +3141,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-array",
  "rand 0.9.2",
@@ -4261,7 +4261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
 dependencies = [
  "equivalent",
- "hashbrown 0.15.5",
+ "hashbrown 0.16.0",
  "serde",
  "serde_core",
 ]
@@ -4478,8 +4478,8 @@ dependencies = [
 
 [[package]]
 name = "lance"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4544,8 +4544,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4563,8 +4563,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrayref",
  "paste",
@@ -4573,8 +4573,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4610,8 +4610,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4641,8 +4641,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4659,8 +4659,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4697,8 +4697,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -4730,8 +4730,8 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "datafusion",
  "geo-types",
@@ -4742,8 +4742,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4804,8 +4804,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -4845,8 +4845,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4862,8 +4862,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "async-trait",
@@ -4875,8 +4875,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -4884,6 +4884,7 @@ dependencies = [
  "async-trait",
  "axum",
  "bytes",
+ "chrono",
  "futures",
  "lance",
  "lance-core",
@@ -4905,9 +4906,9 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-reqwest-client"
-version = "0.0.18"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ea349999bcda4eea53fc05d334b3775ec314761e6a706555c777d7a29b18d19"
+checksum = "a2acdba67f84190067532fce07b51a435dd390d7cdc1129a05003e5cb3274cf0"
 dependencies = [
  "reqwest",
  "serde",
@@ -4918,8 +4919,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4958,8 +4959,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "1.0.1-beta.1"
-source = "git+https://github.com/lance-format/lance.git?tag=v1.0.1-beta.1#9e65b2a9ca17b1c81a33183e5660f88d1b3b9ce0"
+version = "1.0.4"
+source = "git+https://github.com/lance-format/lance.git?tag=v1.0.4#a93eaad1f6909a843cf8aa00d5530359012a7aaa"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4970,7 +4971,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.23.0"
+version = "0.24.1"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5049,7 +5050,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.23.0"
+version = "0.24.1"
 dependencies = [
  "arrow-array",
  "arrow-ipc",
@@ -5069,7 +5070,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.26.0"
+version = "0.27.1"
 dependencies = [
  "arrow",
  "async-trait",
@@ -6725,8 +6726,8 @@ version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
- "heck 0.4.1",
- "itertools 0.12.1",
+ "heck 0.5.0",
+ "itertools 0.14.0",
  "log",
  "multimap",
  "once_cell",
@@ -6746,7 +6747,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
 dependencies = [
  "anyhow",
- "itertools 0.12.1",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
  "syn 2.0.106",
@@ -8076,7 +8077,7 @@ version = "0.8.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
  "syn 2.0.106",

Cargo.toml

@@ -12,23 +12,23 @@ repository = "https://github.com/lancedb/lancedb"
 description = "Serverless, low-latency vector database for AI applications"
 keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
-rust-version = "1.78.0"
+rust-version = "1.88.0"
 
 [workspace.dependencies]
-lance = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=1.0.1-beta.1", default-features = false, "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=1.0.1-beta.1", "tag" = "v1.0.1-beta.1", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=1.0.4", default-features = false, "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=1.0.4", default-features = false, "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=1.0.4", default-features = false, "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=1.0.4", "tag" = "v1.0.4", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "56.2", optional = false }

ci/run_with_test_connection.sh

@@ -16,7 +16,7 @@ check_command_exists() {
 }
 
 if [[ ! -e ./lancedb ]]; then
-    if [[ -v SOPHON_READ_TOKEN ]]; then
+    if [[ x${SOPHON_READ_TOKEN} != "x" ]]; then
         INPUT="lancedb-linux-x64"
         gh release \
             --repo lancedb/lancedb \

docs/mkdocs.yml

@@ -11,7 +11,7 @@ watch:
 theme:
   name: "material"
   logo: assets/logo.png
-  favicon: assets/logo.png
+  favicon: assets/favicon.ico
   palette:
     # Palette toggle for light mode
     - scheme: lancedb
@@ -32,8 +32,6 @@ theme:
     - content.tooltips
     - toc.follow
     - navigation.top
-    - navigation.tabs
-    - navigation.tabs.sticky
     - navigation.footer
     - navigation.tracking
     - navigation.instant
@@ -115,12 +113,13 @@ markdown_extensions:
       emoji_index: !!python/name:material.extensions.emoji.twemoji
       emoji_generator: !!python/name:material.extensions.emoji.to_svg
   - markdown.extensions.toc:
-      baselevel: 1
-      permalink: ""
+      toc_depth: 3
+      permalink: true
+      permalink_title: Anchor link to this section
 
 nav:
-  - API reference:
-      - Overview: index.md
+  - Documentation:
+      - SDK Reference: index.md
       - Python: python/python.md
       - Javascript/TypeScript: js/globals.md
       - Java: java/java.md

docs/src/embeddings/available_embedding_models/multimodal_embedding_functions/voyageai_multimodal_embedding.md

@@ -0,0 +1,111 @@
+# VoyageAI Embeddings : Multimodal
+
+VoyageAI embeddings can also be used to embed both text and image data, only some of the models support image data and you can check the list
+under [https://docs.voyageai.com/docs/multimodal-embeddings](https://docs.voyageai.com/docs/multimodal-embeddings)
+
+Supported multimodal models:
+
+- `voyage-multimodal-3` - 1024 dimensions (text + images)
+- `voyage-multimodal-3.5` - Flexible dimensions (256, 512, 1024 default, 2048). Supports text, images, and video.
+
+### Video Support (voyage-multimodal-3.5)
+
+The `voyage-multimodal-3.5` model supports video input through:
+- Video URLs (`.mp4`, `.webm`, `.mov`, `.avi`, `.mkv`, `.m4v`, `.gif`)
+- Video file paths
+
+Constraints: Max 20MB video size.
+
+Supported parameters (to be passed in `create` method) are:
+
+| Parameter | Type | Default Value           | Description                               |
+|---|---|-------------------------|-------------------------------------------|
+| `name` | `str` | `"voyage-multimodal-3"` | The model ID of the VoyageAI model to use |
+| `output_dimension` | `int` | `None` | Output dimension for voyage-multimodal-3.5. Valid: 256, 512, 1024, 2048 |
+
+Usage Example:
+
+```python
+import base64
+import os
+from io import BytesIO
+
+import requests
+import lancedb
+from lancedb.pydantic import LanceModel, Vector
+from lancedb.embeddings import get_registry
+import pandas as pd
+
+os.environ['VOYAGE_API_KEY'] = 'YOUR_VOYAGE_API_KEY'
+
+db = lancedb.connect(".lancedb")
+func = get_registry().get("voyageai").create(name="voyage-multimodal-3")
+
+
+def image_to_base64(image_bytes: bytes):
+    buffered = BytesIO(image_bytes)
+    img_str = base64.b64encode(buffered.getvalue())
+    return img_str.decode("utf-8")
+
+
+class Images(LanceModel):
+    label: str
+    image_uri: str = func.SourceField()  # image uri as the source
+    image_bytes: str = func.SourceField()  # image bytes base64 encoded as the source
+    vector: Vector(func.ndims()) = func.VectorField()  # vector column
+    vec_from_bytes: Vector(func.ndims()) = func.VectorField()  # Another vector column
+
+
+if "images" in db.table_names():
+    db.drop_table("images")
+table = db.create_table("images", schema=Images)
+labels = ["cat", "cat", "dog", "dog", "horse", "horse"]
+uris = [
+    "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+    "http://farm1.staticflickr.com/134/332220238_da527d8140_z.jpg",
+    "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+    "http://farm5.staticflickr.com/4092/5017326486_1f46057f5f_z.jpg",
+    "http://farm9.staticflickr.com/8216/8434969557_d37882c42d_z.jpg",
+    "http://farm6.staticflickr.com/5142/5835678453_4f3a4edb45_z.jpg",
+]
+# get each uri as bytes
+images_bytes = [image_to_base64(requests.get(uri).content) for uri in uris]
+table.add(
+    pd.DataFrame({"label": labels, "image_uri": uris, "image_bytes": images_bytes})
+)
+```
+Now we can search using text from both the default vector column and the custom vector column
+```python
+
+# text search
+actual = table.search("man's best friend", "vec_from_bytes").limit(1).to_pydantic(Images)[0]
+print(actual.label) # prints "dog"
+
+frombytes = (
+    table.search("man's best friend", vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(frombytes.label)
+
+```
+
+Because we're using a multi-modal embedding function, we can also search using images
+
+```python
+# image search
+query_image_uri = "http://farm1.staticflickr.com/200/467715466_ed4a31801f_z.jpg"
+image_bytes = requests.get(query_image_uri).content
+query_image = Image.open(BytesIO(image_bytes))
+actual = table.search(query_image, "vec_from_bytes").limit(1).to_pydantic(Images)[0]
+print(actual.label == "dog")
+
+# image search using a custom vector column
+other = (
+    table.search(query_image, vector_column_name="vec_from_bytes")
+    .limit(1)
+    .to_pydantic(Images)[0]
+)
+print(actual.label)
+
+```

docs/src/index.md

@@ -1,8 +1,12 @@
-# API Reference
+# SDK Reference
 
-This page contains the API reference for the SDKs supported by the LanceDB team.
+This site contains the API reference for the client SDKs supported by [LanceDB](https://lancedb.com).
 
 - [Python](python/python.md)
 - [JavaScript/TypeScript](js/globals.md)
 - [Java](java/java.md)
-- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)
+- [Rust](https://docs.rs/lancedb/latest/lancedb/index.html)
+
+!!! info "LanceDB Documentation"
+
+    If you're looking for the full documentation of LanceDB, visit [docs.lancedb.com](https://docs.lancedb.com).

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.23.0</version>
+    <version>0.24.1</version>
 </dependency>
 ```
 

docs/src/styles/extra.css

@@ -85,17 +85,26 @@
 
 /* Header gradient (only header area) */
 .md-header {
-  background: linear-gradient(90deg, #3B2E58 0%, #F0B7C1 45%, #E55A2B 100%);
+  background: linear-gradient(90deg, #e4d8f8 0%, #F0B7C1 45%, #E55A2B 100%);
   box-shadow: inset 0 1px 0 rgba(255,255,255,0.08), 0 1px 0 rgba(0,0,0,0.08);
 }
 
+/* Improve brand title contrast on the lavender side */
+.md-header__title,
+.md-header__topic,
+.md-header__title .md-ellipsis,
+.md-header__topic .md-ellipsis {
+  color: #2b1b3a;
+  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.25);
+}
+
 /* Same colors as header for tabs (that hold the text) */
 .md-tabs {
-  background: linear-gradient(90deg, #3B2E58 0%, #F0B7C1 45%, #E55A2B 100%);
+  background: linear-gradient(90deg, #e4d8f8 0%, #F0B7C1 45%, #E55A2B 100%);
 }
 
 /* Dark scheme variant */
 [data-md-color-scheme="slate"] .md-header,
 [data-md-color-scheme="slate"] .md-tabs {
-  background: linear-gradient(90deg, #3B2E58 0%, #F0B7C1 45%, #E55A2B 100%);
+  background: linear-gradient(90deg, #e4d8f8 0%, #F0B7C1 45%, #E55A2B 100%);
 }

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.23.0-final.0</version>
+      <version>0.24.1-final.0</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.23.0-final.0</version>
+    <version>0.24.1-final.0</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.23.0"
+version = "0.24.1"
 license.workspace = true
 description.workspace = true
 repository.workspace = true
@@ -36,6 +36,6 @@ aws-lc-rs = "=1.13.0"
 napi-build = "2.1"
 
 [features]
-default = ["remote", "lancedb/default"]
+default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]

nodejs/lancedb/table.ts

@@ -538,6 +538,35 @@ export abstract class Table {
    *
    */
   abstract stats(): Promise<TableStatistics>;
+
+  /**
+   * Get the initial storage options that were passed in when opening this table.
+   *
+   * For dynamically refreshed options (e.g., credential vending), use
+   * {@link Table.latestStorageOptions}.
+   *
+   * Warning: This is an internal API and the return value is subject to change.
+   *
+   * @returns The storage options, or undefined if no storage options were configured.
+   */
+  abstract initialStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  >;
+
+  /**
+   * Get the latest storage options, refreshing from provider if configured.
+   *
+   * This method is useful for credential vending scenarios where storage options
+   * may be refreshed dynamically. If no dynamic provider is configured, this
+   * returns the initial static options.
+   *
+   * Warning: This is an internal API and the return value is subject to change.
+   *
+   * @returns The storage options, or undefined if no storage options were configured.
+   */
+  abstract latestStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  >;
 }
 
 export class LocalTable extends Table {
@@ -858,6 +887,18 @@ export class LocalTable extends Table {
     return await this.inner.stats();
   }
 
+  async initialStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  > {
+    return await this.inner.initialStorageOptions();
+  }
+
+  async latestStorageOptions(): Promise<
+    Record<string, string> | null | undefined
+  > {
+    return await this.inner.latestStorageOptions();
+  }
+
   mergeInsert(on: string | string[]): MergeInsertBuilder {
     on = Array.isArray(on) ? on : [on];
     return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema());

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/darwin-x64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-x64",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["darwin"],
 	"cpu": ["x64"],
 	"main": "lancedb.darwin-x64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.23.0",
+  "version": "0.24.1",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.23.0",
+	"version": "0.24.1",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.23.0",
+  "version": "0.24.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.23.0",
+      "version": "0.24.1",
       "cpu": [
         "x64",
         "arm64"

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.23.0",
+  "version": "0.24.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

nodejs/src/table.rs

@@ -166,6 +166,19 @@ impl Table {
         Ok(stats.into())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn initial_storage_options(&self) -> napi::Result<Option<HashMap<String, String>>> {
+        Ok(self.inner_ref()?.initial_storage_options().await)
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn latest_storage_options(&self) -> napi::Result<Option<HashMap<String, String>>> {
+        self.inner_ref()?
+            .latest_storage_options()
+            .await
+            .default_error()
+    }
+
     #[napi(catch_unwind)]
     pub async fn update(
         &self,

python/.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.26.1-beta.0"
+current_version = "0.27.1"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

python/Cargo.toml

@@ -1,13 +1,13 @@
 [package]
 name = "lancedb-python"
-version = "0.26.1-beta.0"
+version = "0.27.1"
 edition.workspace = true
 description = "Python bindings for LanceDB"
 license.workspace = true
 repository.workspace = true
 keywords.workspace = true
 categories.workspace = true
-rust-version = "1.75.0"
+rust-version = "1.88.0"
 
 [lib]
 name = "_lancedb"
@@ -38,6 +38,6 @@ pyo3-build-config = { version = "0.25", features = [
 ] }
 
 [features]
-default = ["remote", "lancedb/default"]
+default = ["remote",  "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface"]
 fp16kernels = ["lancedb/fp16kernels"]
 remote = ["lancedb/remote"]

python/python/lancedb/__init__.py

@@ -13,6 +13,7 @@ __version__ = importlib.metadata.version("lancedb")
 
 from ._lancedb import connect as lancedb_connect
 from .common import URI, sanitize_uri
+from urllib.parse import urlparse
 from .db import AsyncConnection, DBConnection, LanceDBConnection
 from .io import StorageOptionsProvider
 from .remote import ClientConfig
@@ -28,6 +29,39 @@ from .namespace import (
 )
 
 
+def _check_s3_bucket_with_dots(
+    uri: str, storage_options: Optional[Dict[str, str]]
+) -> None:
+    """
+    Check if an S3 URI has a bucket name containing dots and warn if no region
+    is specified. S3 buckets with dots cannot use virtual-hosted-style URLs,
+    which breaks automatic region detection.
+
+    See: https://github.com/lancedb/lancedb/issues/1898
+    """
+    if not isinstance(uri, str) or not uri.startswith("s3://"):
+        return
+
+    parsed = urlparse(uri)
+    bucket = parsed.netloc
+
+    if "." not in bucket:
+        return
+
+    # Check if region is provided in storage_options
+    region_keys = {"region", "aws_region"}
+    has_region = storage_options and any(k in storage_options for k in region_keys)
+
+    if not has_region:
+        raise ValueError(
+            f"S3 bucket name '{bucket}' contains dots, which prevents automatic "
+            f"region detection. Please specify the region explicitly via "
+            f"storage_options={{'region': '<your-region>'}} or "
+            f"storage_options={{'aws_region': '<your-region>'}}. "
+            f"See https://github.com/lancedb/lancedb/issues/1898 for details."
+        )
+
+
 def connect(
     uri: URI,
     *,
@@ -121,9 +155,11 @@ def connect(
             storage_options=storage_options,
             **kwargs,
         )
+    _check_s3_bucket_with_dots(str(uri), storage_options)
 
     if kwargs:
         raise ValueError(f"Unknown keyword arguments: {kwargs}")
+
     return LanceDBConnection(
         uri,
         read_consistency_interval=read_consistency_interval,
@@ -211,6 +247,8 @@ async def connect_async(
     if isinstance(client_config, dict):
         client_config = ClientConfig(**client_config)
 
+    _check_s3_bucket_with_dots(str(uri), storage_options)
+
     return AsyncConnection(
         await lancedb_connect(
             sanitize_uri(uri),

python/python/lancedb/_lancedb.pyi

@@ -179,6 +179,9 @@ class Table:
         cleanup_since_ms: Optional[int] = None,
         delete_unverified: Optional[bool] = None,
     ) -> OptimizeStats: ...
+    async def uri(self) -> str: ...
+    async def initial_storage_options(self) -> Optional[Dict[str, str]]: ...
+    async def latest_storage_options(self) -> Optional[Dict[str, str]]: ...
     @property
     def tags(self) -> Tags: ...
     def query(self) -> Query: ...

python/python/lancedb/db.py

@@ -210,10 +210,8 @@ class DBConnection(EnforceOverrides):
         page_token: str, optional
             The token to use for pagination. If not present, start from the beginning.
             Typically, this token is last table name from the previous page.
-            Only supported by LanceDb Cloud.
         limit: int, default 10
             The size of the page to return.
-            Only supported by LanceDb Cloud.
 
         Returns
         -------

python/python/lancedb/embeddings/voyageai.py

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 import base64
 import os
-from typing import ClassVar, TYPE_CHECKING, List, Union, Any, Generator
+from typing import ClassVar, TYPE_CHECKING, List, Union, Any, Generator, Optional
 
 from pathlib import Path
 from urllib.parse import urlparse
@@ -45,11 +45,29 @@ def is_valid_url(text):
         return False
 
 
+VIDEO_EXTENSIONS = {".mp4", ".webm", ".mov", ".avi", ".mkv", ".m4v", ".gif"}
+
+
+def is_video_url(url: str) -> bool:
+    """Check if URL points to a video file based on extension."""
+    parsed = urlparse(url)
+    path = parsed.path.lower()
+    return any(path.endswith(ext) for ext in VIDEO_EXTENSIONS)
+
+
+def is_video_path(path: Path) -> bool:
+    """Check if file path is a video file based on extension."""
+    return path.suffix.lower() in VIDEO_EXTENSIONS
+
+
 def transform_input(input_data: Union[str, bytes, Path]):
     PIL = attempt_import_or_raise("PIL", "pillow")
     if isinstance(input_data, str):
         if is_valid_url(input_data):
-            content = {"type": "image_url", "image_url": input_data}
+            if is_video_url(input_data):
+                content = {"type": "video_url", "video_url": input_data}
+            else:
+                content = {"type": "image_url", "image_url": input_data}
         else:
             content = {"type": "text", "text": input_data}
     elif isinstance(input_data, PIL.Image.Image):
@@ -70,14 +88,24 @@ def transform_input(input_data: Union[str, bytes, Path]):
             "image_base64": "data:image/jpeg;base64," + img_str,
         }
     elif isinstance(input_data, Path):
-        img = PIL.Image.open(input_data)
-        buffered = BytesIO()
-        img.save(buffered, format="JPEG")
-        img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
-        content = {
-            "type": "image_base64",
-            "image_base64": "data:image/jpeg;base64," + img_str,
-        }
+        if is_video_path(input_data):
+            # Read video file and encode as base64
+            with open(input_data, "rb") as f:
+                video_bytes = f.read()
+            video_str = base64.b64encode(video_bytes).decode("utf-8")
+            content = {
+                "type": "video_base64",
+                "video_base64": video_str,
+            }
+        else:
+            img = PIL.Image.open(input_data)
+            buffered = BytesIO()
+            img.save(buffered, format="JPEG")
+            img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+            content = {
+                "type": "image_base64",
+                "image_base64": "data:image/jpeg;base64," + img_str,
+            }
     else:
         raise ValueError("Each input should be either str, bytes, Path or Image.")
 
@@ -91,6 +119,8 @@ def sanitize_multimodal_input(inputs: Union[TEXT, IMAGES]) -> List[Any]:
     PIL = attempt_import_or_raise("PIL", "pillow")
     if isinstance(inputs, (str, bytes, Path, PIL.Image.Image)):
         inputs = [inputs]
+    elif isinstance(inputs, list):
+        pass  # Already a list, use as-is
     elif isinstance(inputs, pa.Array):
         inputs = inputs.to_pylist()
     elif isinstance(inputs, pa.ChunkedArray):
@@ -143,11 +173,16 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
             * voyage-3
             * voyage-3-lite
             * voyage-multimodal-3
+            * voyage-multimodal-3.5
             * voyage-finance-2
             * voyage-multilingual-2
             * voyage-law-2
             * voyage-code-2
 
+    output_dimension: int, optional
+        The output dimension for models that support flexible dimensions.
+        Currently only voyage-multimodal-3.5 supports this feature.
+        Valid options: 256, 512, 1024 (default), 2048.
 
     Examples
     --------
@@ -175,7 +210,10 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
     """
 
     name: str
+    output_dimension: Optional[int] = None
     client: ClassVar = None
+    _FLEXIBLE_DIM_MODELS: ClassVar[list] = ["voyage-multimodal-3.5"]
+    _VALID_DIMENSIONS: ClassVar[list] = [256, 512, 1024, 2048]
     text_embedding_models: list = [
         "voyage-3.5",
         "voyage-3.5-lite",
@@ -186,7 +224,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
         "voyage-law-2",
         "voyage-code-2",
     ]
-    multimodal_embedding_models: list = ["voyage-multimodal-3"]
+    multimodal_embedding_models: list = ["voyage-multimodal-3", "voyage-multimodal-3.5"]
     contextual_embedding_models: list = ["voyage-context-3"]
 
     def _is_multimodal_model(self, model_name: str):
@@ -198,6 +236,17 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
         return model_name in self.contextual_embedding_models or "context" in model_name
 
     def ndims(self):
+        # Handle flexible dimension models
+        if self.name in self._FLEXIBLE_DIM_MODELS:
+            if self.output_dimension is not None:
+                if self.output_dimension not in self._VALID_DIMENSIONS:
+                    raise ValueError(
+                        f"Invalid output_dimension {self.output_dimension} "
+                        f"for {self.name}. Valid options: {self._VALID_DIMENSIONS}"
+                    )
+                return self.output_dimension
+            return 1024  # default dimension
+
         if self.name == "voyage-3-lite":
             return 512
         elif self.name == "voyage-code-2":
@@ -211,12 +260,17 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
             "voyage-finance-2",
             "voyage-multilingual-2",
             "voyage-law-2",
-            "voyage-multimodal-3",
         ]:
             return 1024
         else:
             raise ValueError(f"Model {self.name} not supported")
 
+    def _get_multimodal_kwargs(self, **kwargs):
+        """Get kwargs for multimodal embed call, including output_dimension if set."""
+        if self.name in self._FLEXIBLE_DIM_MODELS and self.output_dimension is not None:
+            kwargs["output_dimension"] = self.output_dimension
+        return kwargs
+
     def compute_query_embeddings(
         self, query: Union[str, "PIL.Image.Image"], *args, **kwargs
     ) -> List[np.ndarray]:
@@ -234,6 +288,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
         """
         client = VoyageAIEmbeddingFunction._get_client()
         if self._is_multimodal_model(self.name):
+            kwargs = self._get_multimodal_kwargs(**kwargs)
             result = client.multimodal_embed(
                 inputs=[[query]], model=self.name, input_type="query", **kwargs
             )
@@ -275,6 +330,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
             )
             if has_images:
                 # Use non-batched API for images
+                kwargs = self._get_multimodal_kwargs(**kwargs)
                 result = client.multimodal_embed(
                     inputs=sanitized, model=self.name, input_type="document", **kwargs
                 )
@@ -357,6 +413,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
             callable: A function that takes a batch of texts and returns embeddings.
         """
         if self._is_multimodal_model(self.name):
+            multimodal_kwargs = self._get_multimodal_kwargs(**kwargs)
 
             def embed_batch(batch: List[str]) -> List[np.array]:
                 batch_inputs = sanitize_multimodal_input(batch)
@@ -364,7 +421,7 @@ class VoyageAIEmbeddingFunction(EmbeddingFunction):
                     inputs=batch_inputs,
                     model=self.name,
                     input_type=input_type,
-                    **kwargs,
+                    **multimodal_kwargs,
                 )
                 return result.embeddings
 

python/python/lancedb/remote/db.py

@@ -384,6 +384,7 @@ class RemoteDBConnection(DBConnection):
         on_bad_vectors: str = "error",
         fill_value: float = 0.0,
         mode: Optional[str] = None,
+        exist_ok: bool = False,
         embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
         *,
         namespace: Optional[List[str]] = None,
@@ -412,6 +413,12 @@ class RemoteDBConnection(DBConnection):
             - pyarrow.Schema
 
             - [LanceModel][lancedb.pydantic.LanceModel]
+        mode: str, default "create"
+            The mode to use when creating the table.
+            Can be either "create", "overwrite", or "exist_ok".
+        exist_ok: bool, default False
+            If exist_ok is True, and mode is None or "create", mode will be changed
+            to "exist_ok".
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
             One of "error", "drop", "fill".
@@ -483,6 +490,11 @@ class RemoteDBConnection(DBConnection):
         LanceTable(table4)
 
         """
+        if exist_ok:
+            if mode == "create":
+                mode = "exist_ok"
+            elif not mode:
+                mode = "exist_ok"
         if namespace is None:
             namespace = []
         validate_table_name(name)

python/python/lancedb/remote/table.py

@@ -18,7 +18,17 @@ from lancedb._lancedb import (
     UpdateResult,
 )
 from lancedb.embeddings.base import EmbeddingFunctionConfig
-from lancedb.index import FTS, BTree, Bitmap, HnswSq, IvfFlat, IvfPq, IvfSq, LabelList
+from lancedb.index import (
+    FTS,
+    BTree,
+    Bitmap,
+    HnswSq,
+    IvfFlat,
+    IvfPq,
+    IvfRq,
+    IvfSq,
+    LabelList,
+)
 from lancedb.remote.db import LOOP
 import pyarrow as pa
 
@@ -265,6 +275,12 @@ class RemoteTable(Table):
                 num_sub_vectors=num_sub_vectors,
                 num_bits=num_bits,
             )
+        elif index_type == "IVF_RQ":
+            config = IvfRq(
+                distance_type=metric,
+                num_partitions=num_partitions,
+                num_bits=num_bits,
+            )
         elif index_type == "IVF_SQ":
             config = IvfSq(distance_type=metric, num_partitions=num_partitions)
         elif index_type == "IVF_HNSW_PQ":
@@ -279,7 +295,8 @@ class RemoteTable(Table):
         else:
             raise ValueError(
                 f"Unknown vector index type: {index_type}. Valid options are"
-                " 'IVF_FLAT', 'IVF_SQ', 'IVF_PQ', 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
+                " 'IVF_FLAT', 'IVF_PQ', 'IVF_RQ', 'IVF_SQ',"
+                " 'IVF_HNSW_PQ', 'IVF_HNSW_SQ'"
             )
 
         LOOP.run(
@@ -638,6 +655,14 @@ class RemoteTable(Table):
     def stats(self):
         return LOOP.run(self._table.stats())
 
+    @property
+    def uri(self) -> str:
+        """The table URI (storage location).
+
+        For remote tables, this fetches the location from the server via describe.
+        """
+        return LOOP.run(self._table.uri())
+
     def take_offsets(self, offsets: list[int]) -> LanceTakeQueryBuilder:
         return LanceTakeQueryBuilder(self._table.take_offsets(offsets))
 

python/python/lancedb/table.py

@@ -684,6 +684,24 @@ class Table(ABC):
         """
         raise NotImplementedError
 
+    def to_lance(self, **kwargs) -> lance.LanceDataset:
+        """Return the table as a lance.LanceDataset.
+
+        Returns
+        -------
+        lance.LanceDataset
+        """
+        raise NotImplementedError
+
+    def to_polars(self, **kwargs) -> "pl.DataFrame":
+        """Return the table as a polars.DataFrame.
+
+        Returns
+        -------
+        polars.DataFrame
+        """
+        raise NotImplementedError
+
     def create_index(
         self,
         metric="l2",
@@ -2200,6 +2218,41 @@ class LanceTable(Table):
     def stats(self) -> TableStatistics:
         return LOOP.run(self._table.stats())
 
+    @property
+    def uri(self) -> str:
+        return LOOP.run(self._table.uri())
+
+    def initial_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the initial storage options that were passed in when opening this table.
+
+        For dynamically refreshed options (e.g., credential vending), use
+        :meth:`latest_storage_options`.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return LOOP.run(self._table.initial_storage_options())
+
+    def latest_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the latest storage options, refreshing from provider if configured.
+
+        This method is useful for credential vending scenarios where storage options
+        may be refreshed dynamically. If no dynamic provider is configured, this
+        returns the initial static options.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return LOOP.run(self._table.latest_storage_options())
+
     def create_scalar_index(
         self,
         column: str,
@@ -3588,6 +3641,51 @@ class AsyncTable:
         """
         return await self._inner.stats()
 
+    async def uri(self) -> str:
+        """
+        Get the table URI (storage location).
+
+        For remote tables, this fetches the location from the server via describe.
+        For local tables, this returns the dataset URI.
+
+        Returns
+        -------
+        str
+            The full storage location of the table (e.g., S3/GCS path).
+        """
+        return await self._inner.uri()
+
+    async def initial_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the initial storage options that were passed in when opening this table.
+
+        For dynamically refreshed options (e.g., credential vending), use
+        :meth:`latest_storage_options`.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return await self._inner.initial_storage_options()
+
+    async def latest_storage_options(self) -> Optional[Dict[str, str]]:
+        """Get the latest storage options, refreshing from provider if configured.
+
+        This method is useful for credential vending scenarios where storage options
+        may be refreshed dynamically. If no dynamic provider is configured, this
+        returns the initial static options.
+
+        Warning: This is an internal API and the return value is subject to change.
+
+        Returns
+        -------
+        Optional[Dict[str, str]]
+            The storage options, or None if no storage options were configured.
+        """
+        return await self._inner.latest_storage_options()
+
     async def add(
         self,
         data: DATA,

python/python/tests/conftest.py

@@ -2,12 +2,27 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 from datetime import timedelta
+
 from lancedb.db import AsyncConnection, DBConnection
 import lancedb
 import pytest
 import pytest_asyncio
 
 
+def pandas_string_type():
+    """Return the PyArrow string type that pandas uses for string columns.
+
+    pandas 3.0+ uses large_string for string columns, pandas 2.x uses string.
+    """
+    import pandas as pd
+    import pyarrow as pa
+
+    version = tuple(int(x) for x in pd.__version__.split(".")[:2])
+    if version >= (3, 0):
+        return pa.large_utf8()
+    return pa.utf8()
+
+
 # Use an in-memory database for most tests.
 @pytest.fixture
 def mem_db() -> DBConnection:

python/python/tests/test_db.py

@@ -268,6 +268,8 @@ async def test_create_table_from_iterator_async(mem_db_async: lancedb.AsyncConne
 
 
 def test_create_exist_ok(tmp_db: lancedb.DBConnection):
+    from conftest import pandas_string_type
+
     data = pd.DataFrame(
         {
             "vector": [[3.1, 4.1], [5.9, 26.5]],
@@ -286,10 +288,11 @@ def test_create_exist_ok(tmp_db: lancedb.DBConnection):
     assert tbl.schema == tbl2.schema
     assert len(tbl) == len(tbl2)
 
+    # pandas 3.0+ uses large_string, pandas 2.x uses string
     schema = pa.schema(
         [
             pa.field("vector", pa.list_(pa.float32(), list_size=2)),
-            pa.field("item", pa.utf8()),
+            pa.field("item", pandas_string_type()),
             pa.field("price", pa.float64()),
         ]
     )
@@ -299,7 +302,7 @@ def test_create_exist_ok(tmp_db: lancedb.DBConnection):
     bad_schema = pa.schema(
         [
             pa.field("vector", pa.list_(pa.float32(), list_size=2)),
-            pa.field("item", pa.utf8()),
+            pa.field("item", pandas_string_type()),
             pa.field("price", pa.float64()),
             pa.field("extra", pa.float32()),
         ]
@@ -365,6 +368,8 @@ async def test_create_mode_async(tmp_db_async: lancedb.AsyncConnection):
 
 @pytest.mark.asyncio
 async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection):
+    from conftest import pandas_string_type
+
     data = pd.DataFrame(
         {
             "vector": [[3.1, 4.1], [5.9, 26.5]],
@@ -382,10 +387,11 @@ async def test_create_exist_ok_async(tmp_db_async: lancedb.AsyncConnection):
     assert tbl.name == tbl2.name
     assert await tbl.schema() == await tbl2.schema()
 
+    # pandas 3.0+ uses large_string, pandas 2.x uses string
     schema = pa.schema(
         [
             pa.field("vector", pa.list_(pa.float32(), list_size=2)),
-            pa.field("item", pa.utf8()),
+            pa.field("item", pandas_string_type()),
             pa.field("price", pa.float64()),
         ]
     )
@@ -595,6 +601,8 @@ def test_open_table_sync(tmp_db: lancedb.DBConnection):
 
 @pytest.mark.asyncio
 async def test_open_table(tmp_path):
+    from conftest import pandas_string_type
+
     db = await lancedb.connect_async(tmp_path)
     data = pd.DataFrame(
         {
@@ -614,10 +622,11 @@ async def test_open_table(tmp_path):
         )
         is not None
     )
+    # pandas 3.0+ uses large_string, pandas 2.x uses string
     assert await tbl.schema() == pa.schema(
         {
             "vector": pa.list_(pa.float32(), list_size=2),
-            "item": pa.utf8(),
+            "item": pandas_string_type(),
             "price": pa.float64(),
         }
     )

python/python/tests/test_embeddings_slow.py

@@ -613,6 +613,133 @@ def test_voyageai_multimodal_embedding_text_function():
     assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
 
 
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_multimodal_35_embedding_function():
+    """Test voyage-multimodal-3.5 model with text input."""
+    voyageai = (
+        get_registry()
+        .get("voyageai")
+        .create(name="voyage-multimodal-3.5", max_retries=0)
+    )
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect("~/lancedb")
+    tbl = db.create_table("test_multimodal_35", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == voyageai.ndims()
+    assert voyageai.ndims() == 1024
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_multimodal_35_flexible_dimensions():
+    """Test voyage-multimodal-3.5 model with custom output dimension."""
+    voyageai = (
+        get_registry()
+        .get("voyageai")
+        .create(name="voyage-multimodal-3.5", output_dimension=512, max_retries=0)
+    )
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()
+
+    assert voyageai.ndims() == 512
+
+    df = pd.DataFrame({"text": ["hello world", "goodbye world"]})
+    db = lancedb.connect("~/lancedb")
+    tbl = db.create_table("test_multimodal_35_dim", schema=TextModel, mode="overwrite")
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == 512
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_multimodal_35_image_embedding():
+    """Test voyage-multimodal-3.5 model with image input."""
+    voyageai = (
+        get_registry()
+        .get("voyageai")
+        .create(name="voyage-multimodal-3.5", max_retries=0)
+    )
+
+    class Images(LanceModel):
+        label: str
+        image_uri: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()
+
+    db = lancedb.connect("~/lancedb")
+    table = db.create_table(
+        "test_multimodal_35_images", schema=Images, mode="overwrite"
+    )
+    labels = ["cat", "dog"]
+    uris = [
+        "http://farm1.staticflickr.com/53/167798175_7c7845bbbd_z.jpg",
+        "http://farm9.staticflickr.com/8387/8602747737_2e5c2a45d4_z.jpg",
+    ]
+    table.add(pd.DataFrame({"label": labels, "image_uri": uris}))
+    assert len(table.to_pandas()["vector"][0]) == voyageai.ndims()
+    assert voyageai.ndims() == 1024
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+@pytest.mark.parametrize("dimension", [256, 512, 1024, 2048])
+def test_voyageai_multimodal_35_all_dimensions(dimension):
+    """Test voyage-multimodal-3.5 model with all valid output dimensions."""
+    voyageai = (
+        get_registry()
+        .get("voyageai")
+        .create(name="voyage-multimodal-3.5", output_dimension=dimension, max_retries=0)
+    )
+
+    assert voyageai.ndims() == dimension
+
+    class TextModel(LanceModel):
+        text: str = voyageai.SourceField()
+        vector: Vector(voyageai.ndims()) = voyageai.VectorField()
+
+    df = pd.DataFrame({"text": ["hello world"]})
+    db = lancedb.connect("~/lancedb")
+    tbl = db.create_table(
+        f"test_multimodal_35_dim_{dimension}", schema=TextModel, mode="overwrite"
+    )
+
+    tbl.add(df)
+    assert len(tbl.to_pandas()["vector"][0]) == dimension
+
+
+@pytest.mark.slow
+@pytest.mark.skipif(
+    os.environ.get("VOYAGE_API_KEY") is None, reason="VOYAGE_API_KEY not set"
+)
+def test_voyageai_multimodal_35_invalid_dimension():
+    """Test voyage-multimodal-3.5 model raises error for invalid output dimension."""
+    with pytest.raises(ValueError, match="Invalid output_dimension"):
+        voyageai = (
+            get_registry()
+            .get("voyageai")
+            .create(name="voyage-multimodal-3.5", output_dimension=999, max_retries=0)
+        )
+        # ndims() is where the validation happens
+        voyageai.ndims()
+
+
 @pytest.mark.slow
 @pytest.mark.skipif(
     importlib.util.find_spec("colpali_engine") is None,

python/python/tests/test_namespace_integration.py

@@ -26,6 +26,8 @@ import pytest
 from lance_namespace import (
     CreateEmptyTableRequest,
     CreateEmptyTableResponse,
+    DeclareTableRequest,
+    DeclareTableResponse,
     DescribeTableRequest,
     DescribeTableResponse,
     LanceNamespace,
@@ -160,6 +162,19 @@ class TrackingNamespace(LanceNamespace):
 
         return modified
 
+    def declare_table(self, request: DeclareTableRequest) -> DeclareTableResponse:
+        """Track declare_table calls and inject rotating credentials."""
+        with self.lock:
+            self.create_call_count += 1
+            count = self.create_call_count
+
+        response = self.inner.declare_table(request)
+        response.storage_options = self._modify_storage_options(
+            response.storage_options, count
+        )
+
+        return response
+
     def create_empty_table(
         self, request: CreateEmptyTableRequest
     ) -> CreateEmptyTableResponse:

python/python/tests/test_remote_db.py

@@ -168,6 +168,42 @@ def test_table_len_sync():
         assert len(table) == 1
 
 
+def test_create_table_exist_ok():
+    def handler(request):
+        if request.path == "/v1/table/test/create/?mode=exist_ok":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}], exist_ok=True)
+        assert table is not None
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}], mode="create", exist_ok=True)
+        assert table is not None
+
+
+def test_create_table_exist_ok_with_mode_overwrite():
+    def handler(request):
+        if request.path == "/v1/table/test/create/?mode=overwrite":
+            request.send_response(200)
+            request.send_header("Content-Type", "application/json")
+            request.end_headers()
+            request.wfile.write(b"{}")
+        else:
+            request.send_response(404)
+            request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        table = db.create_table("test", [{"id": 1}], mode="overwrite", exist_ok=True)
+        assert table is not None
+
+
 @pytest.mark.asyncio
 async def test_http_error():
     request_id_holder = {"request_id": None}

python/python/tests/test_s3_bucket_dots.py

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""
+Tests for S3 bucket names containing dots.
+
+Related issue: https://github.com/lancedb/lancedb/issues/1898
+
+These tests validate the early error checking for S3 bucket names with dots.
+No actual S3 connection is made - validation happens before connection.
+"""
+
+import pytest
+import lancedb
+
+# Test URIs
+BUCKET_WITH_DOTS = "s3://my.bucket.name/path"
+BUCKET_WITH_DOTS_AND_REGION = ("s3://my.bucket.name", {"region": "us-east-1"})
+BUCKET_WITH_DOTS_AND_AWS_REGION = ("s3://my.bucket.name", {"aws_region": "us-east-1"})
+BUCKET_WITHOUT_DOTS = "s3://my-bucket/path"
+
+
+class TestS3BucketWithDotsSync:
+    """Tests for connect()."""
+
+    def test_bucket_with_dots_requires_region(self):
+        with pytest.raises(ValueError, match="contains dots"):
+            lancedb.connect(BUCKET_WITH_DOTS)
+
+    def test_bucket_with_dots_and_region_passes(self):
+        uri, opts = BUCKET_WITH_DOTS_AND_REGION
+        db = lancedb.connect(uri, storage_options=opts)
+        assert db is not None
+
+    def test_bucket_with_dots_and_aws_region_passes(self):
+        uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
+        db = lancedb.connect(uri, storage_options=opts)
+        assert db is not None
+
+    def test_bucket_without_dots_passes(self):
+        db = lancedb.connect(BUCKET_WITHOUT_DOTS)
+        assert db is not None
+
+
+class TestS3BucketWithDotsAsync:
+    """Tests for connect_async()."""
+
+    @pytest.mark.asyncio
+    async def test_bucket_with_dots_requires_region(self):
+        with pytest.raises(ValueError, match="contains dots"):
+            await lancedb.connect_async(BUCKET_WITH_DOTS)
+
+    @pytest.mark.asyncio
+    async def test_bucket_with_dots_and_region_passes(self):
+        uri, opts = BUCKET_WITH_DOTS_AND_REGION
+        db = await lancedb.connect_async(uri, storage_options=opts)
+        assert db is not None
+
+    @pytest.mark.asyncio
+    async def test_bucket_with_dots_and_aws_region_passes(self):
+        uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
+        db = await lancedb.connect_async(uri, storage_options=opts)
+        assert db is not None
+
+    @pytest.mark.asyncio
+    async def test_bucket_without_dots_passes(self):
+        db = await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
+        assert db is not None

python/python/tests/test_table.py

@@ -1967,3 +1967,9 @@ def test_add_table_with_empty_embeddings(tmp_path):
         on_bad_vectors="drop",
     )
     assert table.count_rows() == 1
+
+
+def test_table_uri(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("my_table", data=[{"x": 0}])
+    assert table.uri == str(tmp_path / "my_table.lance")

python/python/tests/test_util.py

@@ -528,12 +528,19 @@ def test_sanitize_data(
         else:
             expected_schema = schema
     else:
+        from conftest import pandas_string_type
+
+        # polars uses large_string, pandas 3.0+ uses large_string, others use string
+        if isinstance(data, pl.DataFrame):
+            text_type = pa.large_utf8()
+        elif isinstance(data, pd.DataFrame):
+            text_type = pandas_string_type()
+        else:
+            text_type = pa.string()
         expected_schema = pa.schema(
             {
                 "id": pa.int64(),
-                "text": pa.large_utf8()
-                if isinstance(data, pl.DataFrame)
-                else pa.string(),
+                "text": text_type,
                 "vector": pa.list_(pa.float32(), 10),
             }
         )

python/src/connection.rs

@@ -304,6 +304,7 @@ impl Connection {
                 },
                 page_token,
                 limit: limit.map(|l| l as i32),
+                ..Default::default()
             };
             let response = inner.list_namespaces(request).await.infer_error()?;
             Python::with_gil(|py| -> PyResult<Py<PyDict>> {
@@ -325,11 +326,12 @@ impl Connection {
         let inner = self_.get_inner()?.clone();
         let py = self_.py();
         future_into_py(py, async move {
-            use lance_namespace::models::{create_namespace_request, CreateNamespaceRequest};
-            let mode_enum = mode.and_then(|m| match m.to_lowercase().as_str() {
-                "create" => Some(create_namespace_request::Mode::Create),
-                "exist_ok" => Some(create_namespace_request::Mode::ExistOk),
-                "overwrite" => Some(create_namespace_request::Mode::Overwrite),
+            use lance_namespace::models::CreateNamespaceRequest;
+            // Mode is now a string field
+            let mode_str = mode.and_then(|m| match m.to_lowercase().as_str() {
+                "create" => Some("Create".to_string()),
+                "exist_ok" => Some("ExistOk".to_string()),
+                "overwrite" => Some("Overwrite".to_string()),
                 _ => None,
             });
             let request = CreateNamespaceRequest {
@@ -338,8 +340,9 @@ impl Connection {
                 } else {
                     Some(namespace)
                 },
-                mode: mode_enum,
+                mode: mode_str,
                 properties,
+                ..Default::default()
             };
             let response = inner.create_namespace(request).await.infer_error()?;
             Python::with_gil(|py| -> PyResult<Py<PyDict>> {
@@ -360,15 +363,16 @@ impl Connection {
         let inner = self_.get_inner()?.clone();
         let py = self_.py();
         future_into_py(py, async move {
-            use lance_namespace::models::{drop_namespace_request, DropNamespaceRequest};
-            let mode_enum = mode.and_then(|m| match m.to_uppercase().as_str() {
-                "SKIP" => Some(drop_namespace_request::Mode::Skip),
-                "FAIL" => Some(drop_namespace_request::Mode::Fail),
+            use lance_namespace::models::DropNamespaceRequest;
+            // Mode and Behavior are now string fields
+            let mode_str = mode.and_then(|m| match m.to_uppercase().as_str() {
+                "SKIP" => Some("Skip".to_string()),
+                "FAIL" => Some("Fail".to_string()),
                 _ => None,
             });
-            let behavior_enum = behavior.and_then(|b| match b.to_uppercase().as_str() {
-                "RESTRICT" => Some(drop_namespace_request::Behavior::Restrict),
-                "CASCADE" => Some(drop_namespace_request::Behavior::Cascade),
+            let behavior_str = behavior.and_then(|b| match b.to_uppercase().as_str() {
+                "RESTRICT" => Some("Restrict".to_string()),
+                "CASCADE" => Some("Cascade".to_string()),
                 _ => None,
             });
             let request = DropNamespaceRequest {
@@ -377,8 +381,9 @@ impl Connection {
                 } else {
                     Some(namespace)
                 },
-                mode: mode_enum,
-                behavior: behavior_enum,
+                mode: mode_str,
+                behavior: behavior_str,
+                ..Default::default()
             };
             let response = inner.drop_namespace(request).await.infer_error()?;
             Python::with_gil(|py| -> PyResult<Py<PyDict>> {
@@ -405,6 +410,7 @@ impl Connection {
                 } else {
                     Some(namespace)
                 },
+                ..Default::default()
             };
             let response = inner.describe_namespace(request).await.infer_error()?;
             Python::with_gil(|py| -> PyResult<Py<PyDict>> {
@@ -434,6 +440,7 @@ impl Connection {
                 },
                 page_token,
                 limit: limit.map(|l| l as i32),
+                ..Default::default()
             };
             let response = inner.list_tables(request).await.infer_error()?;
             Python::with_gil(|py| -> PyResult<Py<PyDict>> {

python/src/table.rs

@@ -497,6 +497,25 @@ impl Table {
         })
     }
 
+    pub fn uri(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move { inner.uri().await.infer_error() })
+    }
+
+    pub fn initial_storage_options(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            Ok(inner.initial_storage_options().await)
+        })
+    }
+
+    pub fn latest_storage_options(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            inner.latest_storage_options().await.infer_error()
+        })
+    }
+
     pub fn __repr__(&self) -> String {
         match &self.inner {
             None => format!("ClosedTable({})", self.name),

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.23.0"
+version = "0.24.1"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -104,11 +104,16 @@ test-log = "0.2"
 
 
 [features]
-default = ["aws", "gcs", "azure", "dynamodb", "oss"]
+default = []
 aws = ["lance/aws", "lance-io/aws", "lance-namespace-impls/dir-aws"]
 oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
 gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
 azure = ["lance/azure", "lance-io/azure", "lance-namespace-impls/dir-azure"]
+huggingface = [
+    "lance/huggingface",
+    "lance-io/huggingface",
+    "lance-namespace-impls/dir-huggingface",
+]
 dynamodb = ["lance/dynamodb", "aws"]
 remote = ["dep:reqwest", "dep:http", "lance-namespace-impls/rest", "lance-namespace-impls/rest-adapter"]
 fp16kernels = ["lance-linalg/fp16kernels"]
@@ -148,3 +153,6 @@ name = "ivf_pq"
 [[example]]
 name = "hybrid_search"
 required-features = ["sentence-transformers"]
+
+[package.metadata.docs.rs]
+all-features = true

rust/lancedb/src/connection.rs

@@ -36,10 +36,42 @@ use crate::remote::{
 };
 use crate::table::{TableDefinition, WriteOptions};
 use crate::Table;
+use lance::io::ObjectStoreParams;
 pub use lance_encoding::version::LanceFileVersion;
 #[cfg(feature = "remote")]
 use lance_io::object_store::StorageOptions;
-use lance_io::object_store::StorageOptionsProvider;
+use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+
+fn merge_storage_options(
+    store_params: &mut ObjectStoreParams,
+    pairs: impl IntoIterator<Item = (String, String)>,
+) {
+    let mut options = store_params.storage_options().cloned().unwrap_or_default();
+    for (key, value) in pairs {
+        options.insert(key, value);
+    }
+    let provider = store_params
+        .storage_options_accessor
+        .as_ref()
+        .and_then(|accessor| accessor.provider().cloned());
+    let accessor = if let Some(provider) = provider {
+        StorageOptionsAccessor::with_initial_and_provider(options, provider)
+    } else {
+        StorageOptionsAccessor::with_static_options(options)
+    };
+    store_params.storage_options_accessor = Some(Arc::new(accessor));
+}
+
+fn set_storage_options_provider(
+    store_params: &mut ObjectStoreParams,
+    provider: Arc<dyn StorageOptionsProvider>,
+) {
+    let accessor = match store_params.storage_options().cloned() {
+        Some(options) => StorageOptionsAccessor::with_initial_and_provider(options, provider),
+        None => StorageOptionsAccessor::with_provider(provider),
+    };
+    store_params.storage_options_accessor = Some(Arc::new(accessor));
+}
 
 /// A builder for configuring a [`Connection::table_names`] operation
 pub struct TableNamesBuilder {
@@ -246,16 +278,14 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     ///
     /// See available options at <https://lancedb.com/docs/storage/>
     pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let store_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-        store_options.insert(key.into(), value.into());
+        merge_storage_options(store_params, [(key.into(), value.into())]);
         self
     }
 
@@ -269,19 +299,17 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
         mut self,
         pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
     ) -> Self {
-        let store_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-
-        for (key, value) in pairs {
-            store_options.insert(key.into(), value.into());
-        }
+        let updates = pairs
+            .into_iter()
+            .map(|(key, value)| (key.into(), value.into()));
+        merge_storage_options(store_params, updates);
         self
     }
 
@@ -318,23 +346,21 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// This has no effect in LanceDB Cloud.
     #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
     pub fn enable_v2_manifest_paths(mut self, use_v2_manifest_paths: bool) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert_with(Default::default)
             .store_params
-            .get_or_insert_with(Default::default)
-            .storage_options
             .get_or_insert_with(Default::default);
-
-        storage_options.insert(
-            OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(),
-            if use_v2_manifest_paths {
-                "true".to_string()
-            } else {
-                "false".to_string()
-            },
+        let value = if use_v2_manifest_paths {
+            "true".to_string()
+        } else {
+            "false".to_string()
+        };
+        merge_storage_options(
+            store_params,
+            [(OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(), value)],
         );
         self
     }
@@ -344,19 +370,19 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// The default is `LanceFileVersion::Stable`.
     #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
     pub fn data_storage_version(mut self, data_storage_version: LanceFileVersion) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert_with(Default::default)
             .store_params
-            .get_or_insert_with(Default::default)
-            .storage_options
             .get_or_insert_with(Default::default);
-
-        storage_options.insert(
-            OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
-            data_storage_version.to_string(),
+        merge_storage_options(
+            store_params,
+            [(
+                OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
+                data_storage_version.to_string(),
+            )],
         );
         self
     }
@@ -381,13 +407,14 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// This allows tables to automatically refresh cloud storage credentials
     /// when they expire, enabling long-running operations on remote storage.
     pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        self.request
+        let store_params = self
+            .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default())
-            .storage_options_provider = Some(provider);
+            .get_or_insert(Default::default());
+        set_storage_options_provider(store_params, provider);
         self
     }
 }
@@ -450,15 +477,13 @@ impl OpenTableBuilder {
     ///
     /// See available options at <https://lancedb.com/docs/storage/>
     pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-        storage_options.insert(key.into(), value.into());
+        merge_storage_options(store_params, [(key.into(), value.into())]);
         self
     }
 
@@ -472,18 +497,16 @@ impl OpenTableBuilder {
         mut self,
         pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
     ) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-
-        for (key, value) in pairs {
-            storage_options.insert(key.into(), value.into());
-        }
+        let updates = pairs
+            .into_iter()
+            .map(|(key, value)| (key.into(), value.into()));
+        merge_storage_options(store_params, updates);
         self
     }
 
@@ -507,12 +530,13 @@ impl OpenTableBuilder {
     /// This allows tables to automatically refresh cloud storage credentials
     /// when they expire, enabling long-running operations on remote storage.
     pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        self.request
+        let store_params = self
+            .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default())
-            .storage_options_provider = Some(provider);
+            .get_or_insert(Default::default());
+        set_storage_options_provider(store_params, provider);
         self
     }
 
@@ -868,6 +892,10 @@ pub struct ConnectBuilder {
     embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
 }
 
+#[cfg(feature = "remote")]
+const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 1] =
+    [("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name")];
+
 impl ConnectBuilder {
     /// Create a new [`ConnectOptions`] with the given database URI.
     pub fn new(uri: &str) -> Self {
@@ -1051,11 +1079,27 @@ impl ConnectBuilder {
         self
     }
 
+    #[cfg(feature = "remote")]
+    fn apply_env_defaults(
+        env_var_to_remote_storage_option: &[(&str, &str)],
+        options: &mut HashMap<String, String>,
+    ) {
+        for (env_key, opt_key) in env_var_to_remote_storage_option {
+            if let Ok(env_value) = std::env::var(env_key) {
+                if !options.contains_key(*opt_key) {
+                    options.insert((*opt_key).to_string(), env_value);
+                }
+            }
+        }
+    }
+
     #[cfg(feature = "remote")]
     fn execute_remote(self) -> Result<Connection> {
         use crate::remote::db::RemoteDatabaseOptions;
 
-        let options = RemoteDatabaseOptions::parse_from_map(&self.request.options)?;
+        let mut merged_options = self.request.options.clone();
+        Self::apply_env_defaults(&ENV_VARS_TO_STORAGE_OPTS, &mut merged_options);
+        let options = RemoteDatabaseOptions::parse_from_map(&merged_options)?;
 
         let region = options.region.ok_or_else(|| Error::InvalidInput {
             message: "A region is required when connecting to LanceDb Cloud".to_string(),
@@ -1277,8 +1321,6 @@ mod test_utils {
 
 #[cfg(test)]
 mod tests {
-    use std::fs::create_dir_all;
-
     use crate::database::listing::{ListingDatabaseOptions, NewTableConfig};
     use crate::query::QueryBase;
     use crate::query::{ExecutableQuery, QueryExecutionOptions};
@@ -1302,6 +1344,23 @@ mod tests {
         assert_eq!(tc.connection.uri(), tc.uri);
     }
 
+    #[cfg(feature = "remote")]
+    #[test]
+    fn test_apply_env_defaults() {
+        let env_key = "TEST_APPLY_ENV_DEFAULTS_ENVIRONMENT_VARIABLE_ENV_KEY";
+        let env_val = "TEST_APPLY_ENV_DEFAULTS_ENVIRONMENT_VARIABLE_ENV_VAL";
+        let opts_key = "test_apply_env_defaults_environment_variable_opts_key";
+        std::env::set_var(env_key, env_val);
+
+        let mut options = HashMap::new();
+        ConnectBuilder::apply_env_defaults(&[(env_key, opts_key)], &mut options);
+        assert_eq!(Some(&env_val.to_string()), options.get(opts_key));
+
+        options.insert(opts_key.to_string(), "EXPLICIT-VALUE".to_string());
+        ConnectBuilder::apply_env_defaults(&[(env_key, opts_key)], &mut options);
+        assert_eq!(Some(&"EXPLICIT-VALUE".to_string()), options.get(opts_key));
+    }
+
     #[cfg(not(windows))]
     #[tokio::test]
     async fn test_connect_relative() {
@@ -1325,25 +1384,27 @@ mod tests {
 
     #[tokio::test]
     async fn test_table_names() {
-        let tmp_dir = tempdir().unwrap();
+        let tc = new_test_connection().await.unwrap();
+        let db = tc.connection;
+        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
         let mut names = Vec::with_capacity(100);
         for _ in 0..100 {
-            let mut name = uuid::Uuid::new_v4().to_string();
+            let name = uuid::Uuid::new_v4().to_string();
             names.push(name.clone());
-            name.push_str(".lance");
-            create_dir_all(tmp_dir.path().join(&name)).unwrap();
+            db.create_empty_table(name, schema.clone())
+                .execute()
+                .await
+                .unwrap();
         }
         names.sort();
-
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = connect(uri).execute().await.unwrap();
-        let tables = db.table_names().execute().await.unwrap();
+        let tables = db.table_names().limit(100).execute().await.unwrap();
 
         assert_eq!(tables, names);
 
         let tables = db
             .table_names()
             .start_after(&names[30])
+            .limit(100)
             .execute()
             .await
             .unwrap();
@@ -1524,18 +1585,27 @@ mod tests {
 
     #[tokio::test]
     async fn drop_table() {
-        let tmp_dir = tempdir().unwrap();
+        let tc = new_test_connection().await.unwrap();
+        let db = tc.connection;
 
-        let uri = tmp_dir.path().to_str().unwrap();
-        let db = connect(uri).execute().await.unwrap();
+        if tc.is_remote {
+            // All the typical endpoints such as s3:///, file-object-store:///, etc. treat drop_table
+            // as idempotent.
+            assert!(db.drop_table("invalid_table", &[]).await.is_ok());
+        } else {
+            // The behavior of drop_table when using a file:/// endpoint differs from all other
+            // object providers, in that it returns an error when deleting a non-existent table.
+            assert!(matches!(
+                db.drop_table("invalid_table", &[]).await,
+                Err(crate::Error::TableNotFound { .. }),
+            ));
+        }
 
-        // drop non-exist table
-        assert!(matches!(
-            db.drop_table("invalid_table", &[]).await,
-            Err(crate::Error::TableNotFound { .. }),
-        ));
-
-        create_dir_all(tmp_dir.path().join("table1.lance")).unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
+        db.create_empty_table("table1", schema.clone())
+            .execute()
+            .await
+            .unwrap();
         db.drop_table("table1", &[]).await.unwrap();
 
         let tables = db.table_names().execute().await.unwrap();

rust/lancedb/src/database/listing.rs

@@ -12,7 +12,7 @@ use lance::dataset::{builder::DatasetBuilder, ReadParams, WriteMode};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
-use lance_io::object_store::StorageOptionsProvider;
+use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -356,7 +356,13 @@ impl ListingDatabase {
                     .clone()
                     .unwrap_or_else(|| Arc::new(lance::session::Session::default()));
                 let os_params = ObjectStoreParams {
-                    storage_options: Some(options.storage_options.clone()),
+                    storage_options_accessor: if options.storage_options.is_empty() {
+                        None
+                    } else {
+                        Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                            options.storage_options.clone(),
+                        )))
+                    },
                     ..Default::default()
                 };
                 let (object_store, base_path) = ObjectStore::from_uri_and_params(
@@ -463,9 +469,20 @@ impl ListingDatabase {
         validate_table_name(name)?;
 
         let mut uri = self.uri.clone();
-        // If the URI does not end with a slash, add one
-        if !uri.ends_with('/') {
-            uri.push('/');
+        // If the URI does not end with a path separator, add one
+        // Use forward slash for URIs (http://, s3://, gs://, file://, etc.)
+        // Use platform-specific separator for local paths without scheme
+        let has_scheme = uri.contains("://");
+        let ends_with_separator = uri.ends_with('/') || uri.ends_with('\\');
+
+        if !ends_with_separator {
+            if has_scheme {
+                // URIs always use forward slash
+                uri.push('/');
+            } else {
+                // Local path without scheme - use platform separator
+                uri.push(std::path::MAIN_SEPARATOR);
+            }
         }
         // Append the table name with the lance file extension
         uri.push_str(&format!("{}.{}", name, LANCE_FILE_EXTENSION));
@@ -481,7 +498,13 @@ impl ListingDatabase {
 
     async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
         let object_store_params = ObjectStoreParams {
-            storage_options: Some(self.storage_options.clone()),
+            storage_options_accessor: if self.storage_options.is_empty() {
+                None
+            } else {
+                Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                    self.storage_options.clone(),
+                )))
+            },
             ..Default::default()
         };
         let mut uri = self.uri.clone();
@@ -530,7 +553,7 @@ impl ListingDatabase {
             .lance_write_params
             .as_ref()
             .and_then(|p| p.store_params.as_ref())
-            .and_then(|sp| sp.storage_options.as_ref());
+            .and_then(|sp| sp.storage_options());
 
         let storage_version_override = storage_options
             .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
@@ -581,21 +604,20 @@ impl ListingDatabase {
         // will cause a new connection to be created, and that connection will
         // be dropped from the cache when python GCs the table object, which
         // confounds reuse across tables.
-        if !self.storage_options.is_empty() {
-            let storage_options = write_params
+        if !self.storage_options.is_empty() || self.storage_options_provider.is_some() {
+            let store_params = write_params
                 .store_params
-                .get_or_insert_with(Default::default)
-                .storage_options
                 .get_or_insert_with(Default::default);
-            self.inherit_storage_options(storage_options);
-        }
-
-        // Set storage options provider if available
-        if self.storage_options_provider.is_some() {
-            write_params
-                .store_params
-                .get_or_insert_with(Default::default)
-                .storage_options_provider = self.storage_options_provider.clone();
+            let mut storage_options = store_params.storage_options().cloned().unwrap_or_default();
+            if !self.storage_options.is_empty() {
+                self.inherit_storage_options(&mut storage_options);
+            }
+            let accessor = if let Some(ref provider) = self.storage_options_provider {
+                StorageOptionsAccessor::with_initial_and_provider(storage_options, provider.clone())
+            } else {
+                StorageOptionsAccessor::with_static_options(storage_options)
+            };
+            store_params.storage_options_accessor = Some(Arc::new(accessor));
         }
 
         write_params.data_storage_version = self
@@ -881,7 +903,13 @@ impl Database for ListingDatabase {
         validate_table_name(&request.target_table_name)?;
 
         let storage_params = ObjectStoreParams {
-            storage_options: Some(self.storage_options.clone()),
+            storage_options_accessor: if self.storage_options.is_empty() {
+                None
+            } else {
+                Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                    self.storage_options.clone(),
+                )))
+            },
             ..Default::default()
         };
         let read_params = ReadParams {
@@ -945,25 +973,28 @@ impl Database for ListingDatabase {
         // will cause a new connection to be created, and that connection will
         // be dropped from the cache when python GCs the table object, which
         // confounds reuse across tables.
-        if !self.storage_options.is_empty() {
-            let storage_options = request
+        if !self.storage_options.is_empty() || self.storage_options_provider.is_some() {
+            let store_params = request
                 .lance_read_params
                 .get_or_insert_with(Default::default)
                 .store_options
-                .get_or_insert_with(Default::default)
-                .storage_options
                 .get_or_insert_with(Default::default);
-            self.inherit_storage_options(storage_options);
-        }
-
-        // Set storage options provider if available
-        if self.storage_options_provider.is_some() {
-            request
-                .lance_read_params
-                .get_or_insert_with(Default::default)
-                .store_options
-                .get_or_insert_with(Default::default)
-                .storage_options_provider = self.storage_options_provider.clone();
+            let mut storage_options = store_params.storage_options().cloned().unwrap_or_default();
+            if !self.storage_options.is_empty() {
+                self.inherit_storage_options(&mut storage_options);
+            }
+            // Preserve request-level provider if no connection-level provider exists
+            let request_provider = store_params
+                .storage_options_accessor
+                .as_ref()
+                .and_then(|a| a.provider().cloned());
+            let provider = self.storage_options_provider.clone().or(request_provider);
+            let accessor = if let Some(provider) = provider {
+                StorageOptionsAccessor::with_initial_and_provider(storage_options, provider)
+            } else {
+                StorageOptionsAccessor::with_static_options(storage_options)
+            };
+            store_params.storage_options_accessor = Some(Arc::new(accessor));
         }
 
         // Some ReadParams are exposed in the OpenTableBuilder, but we also
@@ -1071,6 +1102,7 @@ mod tests {
     use crate::table::{Table, TableDefinition};
     use arrow_array::{Int32Array, RecordBatch, StringArray};
     use arrow_schema::{DataType, Field, Schema};
+    use std::path::PathBuf;
     use tempfile::tempdir;
 
     async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
@@ -1869,7 +1901,9 @@ mod tests {
         let write_options = WriteOptions {
             lance_write_params: Some(lance::dataset::WriteParams {
                 store_params: Some(lance::io::ObjectStoreParams {
-                    storage_options: Some(storage_options),
+                    storage_options_accessor: Some(Arc::new(
+                        StorageOptionsAccessor::with_static_options(storage_options),
+                    )),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -1943,7 +1977,9 @@ mod tests {
         let write_options = WriteOptions {
             lance_write_params: Some(lance::dataset::WriteParams {
                 store_params: Some(lance::io::ObjectStoreParams {
-                    storage_options: Some(storage_options),
+                    storage_options_accessor: Some(Arc::new(
+                        StorageOptionsAccessor::with_static_options(storage_options),
+                    )),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -2046,6 +2082,19 @@ mod tests {
         assert_eq!(db_options.new_table_config.enable_stable_row_ids, None);
     }
 
+    #[tokio::test]
+    async fn test_table_uri() {
+        let (_tempdir, db) = setup_database().await;
+
+        let mut pb = PathBuf::new();
+        pb.push(db.uri.clone());
+        pb.push("test.lance");
+
+        let expected = pb.to_str().unwrap();
+        let uri = db.table_uri("test").ok().unwrap();
+        assert_eq!(uri, expected);
+    }
+
     #[tokio::test]
     async fn test_namespace_client() {
         let (_tempdir, db) = setup_database().await;

rust/lancedb/src/database/namespace.rs

@@ -7,25 +7,25 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use async_trait::async_trait;
-use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsProvider};
 use lance_namespace::{
     models::{
         CreateEmptyTableRequest, CreateNamespaceRequest, CreateNamespaceResponse,
-        DescribeNamespaceRequest, DescribeNamespaceResponse, DescribeTableRequest,
-        DropNamespaceRequest, DropNamespaceResponse, DropTableRequest, ListNamespacesRequest,
-        ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
+        DeclareTableRequest, DescribeNamespaceRequest, DescribeNamespaceResponse,
+        DescribeTableRequest, DropNamespaceRequest, DropNamespaceResponse, DropTableRequest,
+        ListNamespacesRequest, ListNamespacesResponse, ListTablesRequest, ListTablesResponse,
     },
     LanceNamespace,
 };
 use lance_namespace_impls::ConnectBuilder;
+use log::warn;
 
-use crate::connection::ConnectRequest;
 use crate::database::ReadConsistency;
 use crate::error::{Error, Result};
+use crate::table::NativeTable;
 
 use super::{
-    listing::ListingDatabase, BaseTable, CloneTableRequest, CreateTableMode,
-    CreateTableRequest as DbCreateTableRequest, Database, OpenTableRequest, TableNamesRequest,
+    BaseTable, CloneTableRequest, CreateTableMode, CreateTableRequest as DbCreateTableRequest,
+    Database, OpenTableRequest, TableNamesRequest,
 };
 
 /// A database implementation that uses lance-namespace for table management
@@ -90,51 +90,6 @@ impl std::fmt::Display for LanceNamespaceDatabase {
     }
 }
 
-impl LanceNamespaceDatabase {
-    /// Create a temporary listing database for the given location
-    ///
-    /// Merges storage options with priority: connection < user < namespace
-    async fn create_listing_database(
-        &self,
-        location: &str,
-        table_id: Vec<String>,
-        user_storage_options: Option<&HashMap<String, String>>,
-        response_storage_options: Option<&HashMap<String, String>>,
-    ) -> Result<ListingDatabase> {
-        // Merge storage options: connection < user < namespace
-        let mut merged_storage_options = self.storage_options.clone();
-        if let Some(opts) = user_storage_options {
-            merged_storage_options.extend(opts.clone());
-        }
-        if let Some(opts) = response_storage_options {
-            merged_storage_options.extend(opts.clone());
-        }
-
-        let request = ConnectRequest {
-            uri: location.to_string(),
-            #[cfg(feature = "remote")]
-            client_config: Default::default(),
-            options: merged_storage_options,
-            read_consistency_interval: self.read_consistency_interval,
-            session: self.session.clone(),
-        };
-
-        let mut listing_db = ListingDatabase::connect_with_options(&request).await?;
-
-        // Create storage options provider only if namespace returned storage options
-        // (not just user-provided options)
-        if response_storage_options.is_some() {
-            let provider = Arc::new(LanceNamespaceStorageOptionsProvider::new(
-                self.namespace.clone(),
-                table_id,
-            )) as Arc<dyn StorageOptionsProvider>;
-            listing_db.storage_options_provider = Some(provider);
-        }
-
-        Ok(listing_db)
-    }
-}
-
 #[async_trait]
 impl Database for LanceNamespaceDatabase {
     fn uri(&self) -> &str {
@@ -183,6 +138,7 @@ impl Database for LanceNamespaceDatabase {
             id: Some(request.namespace),
             page_token: request.start_after,
             limit: request.limit.map(|l| l as i32),
+            ..Default::default()
         };
 
         let response = self.namespace.list_tables(ns_request).await?;
@@ -195,19 +151,11 @@ impl Database for LanceNamespaceDatabase {
     }
 
     async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
-        // Extract user-provided storage options from request
-        let user_storage_options = request
-            .write_options
-            .lance_write_params
-            .as_ref()
-            .and_then(|lwp| lwp.store_params.as_ref())
-            .and_then(|sp| sp.storage_options.as_ref());
-
         let mut table_id = request.namespace.clone();
         table_id.push(request.name.clone());
         let describe_request = DescribeTableRequest {
             id: Some(table_id.clone()),
-            version: None,
+            ..Default::default()
         };
 
         let describe_result = self.namespace.describe_table(describe_request).await;
@@ -225,6 +173,7 @@ impl Database for LanceNamespaceDatabase {
                     // Drop the existing table - must succeed
                     let drop_request = DropTableRequest {
                         id: Some(table_id.clone()),
+                        ..Default::default()
                     };
                     self.namespace
                         .drop_table(drop_request)
@@ -235,34 +184,20 @@ impl Database for LanceNamespaceDatabase {
                 }
             }
             CreateTableMode::ExistOk(_) => {
-                if let Ok(response) = describe_result {
-                    let location = response.location.ok_or_else(|| Error::Runtime {
-                        message: "Table location is missing from namespace response".to_string(),
-                    })?;
+                if describe_result.is_ok() {
+                    let native_table = NativeTable::open_from_namespace(
+                        self.namespace.clone(),
+                        &request.name,
+                        request.namespace.clone(),
+                        None,
+                        None,
+                        self.read_consistency_interval,
+                        self.server_side_query_enabled,
+                        self.session.clone(),
+                    )
+                    .await?;
 
-                    let listing_db = self
-                        .create_listing_database(
-                            &location,
-                            table_id.clone(),
-                            user_storage_options,
-                            response.storage_options.as_ref(),
-                        )
-                        .await?;
-
-                    let namespace_client = self
-                        .server_side_query_enabled
-                        .then(|| self.namespace.clone());
-
-                    return listing_db
-                        .open_table(OpenTableRequest {
-                            name: request.name.clone(),
-                            namespace: request.namespace.clone(),
-                            index_cache_size: None,
-                            lance_read_params: None,
-                            location: Some(location),
-                            namespace_client,
-                        })
-                        .await;
+                    return Ok(Arc::new(native_table));
                 }
             }
         }
@@ -270,106 +205,85 @@ impl Database for LanceNamespaceDatabase {
         let mut table_id = request.namespace.clone();
         table_id.push(request.name.clone());
 
-        let create_empty_request = CreateEmptyTableRequest {
+        // Try declare_table first, falling back to create_empty_table for backwards
+        // compatibility with older namespace clients that don't support declare_table
+        let declare_request = DeclareTableRequest {
             id: Some(table_id.clone()),
-            location: None,
-            properties: if self.storage_options.is_empty() {
-                None
-            } else {
-                Some(self.storage_options.clone())
-            },
+            ..Default::default()
         };
 
-        let create_empty_response = self
-            .namespace
-            .create_empty_table(create_empty_request)
-            .await
-            .map_err(|e| Error::Runtime {
-                message: format!("Failed to create empty table: {}", e),
-            })?;
+        let location = match self.namespace.declare_table(declare_request).await {
+            Ok(response) => response.location.ok_or_else(|| Error::Runtime {
+                message: "Table location is missing from declare_table response".to_string(),
+            })?,
+            Err(e) => {
+                // Check if the error is "not supported" and try create_empty_table as fallback
+                let err_str = e.to_string().to_lowercase();
+                if err_str.contains("not supported") || err_str.contains("not implemented") {
+                    warn!(
+                        "declare_table is not supported by the namespace client, \
+                        falling back to deprecated create_empty_table. \
+                        create_empty_table is deprecated and will be removed in Lance 3.0.0. \
+                        Please upgrade your namespace client to support declare_table."
+                    );
+                    #[allow(deprecated)]
+                    let create_empty_request = CreateEmptyTableRequest {
+                        id: Some(table_id.clone()),
+                        ..Default::default()
+                    };
 
-        let location = create_empty_response
-            .location
-            .ok_or_else(|| Error::Runtime {
-                message: "Table location is missing from create_empty_table response".to_string(),
-            })?;
+                    #[allow(deprecated)]
+                    let create_response = self
+                        .namespace
+                        .create_empty_table(create_empty_request)
+                        .await
+                        .map_err(|e| Error::Runtime {
+                            message: format!("Failed to create empty table: {}", e),
+                        })?;
 
-        let listing_db = self
-            .create_listing_database(
-                &location,
-                table_id.clone(),
-                user_storage_options,
-                create_empty_response.storage_options.as_ref(),
-            )
-            .await?;
-
-        let namespace_client = self
-            .server_side_query_enabled
-            .then(|| self.namespace.clone());
-
-        let create_request = DbCreateTableRequest {
-            name: request.name,
-            namespace: request.namespace,
-            data: request.data,
-            mode: request.mode,
-            write_options: request.write_options,
-            location: Some(location),
-            namespace_client,
+                    create_response.location.ok_or_else(|| Error::Runtime {
+                        message: "Table location is missing from create_empty_table response"
+                            .to_string(),
+                    })?
+                } else {
+                    return Err(Error::Runtime {
+                        message: format!("Failed to declare table: {}", e),
+                    });
+                }
+            }
         };
 
-        listing_db.create_table(create_request).await
+        let native_table = NativeTable::create_from_namespace(
+            self.namespace.clone(),
+            &location,
+            &request.name,
+            request.namespace.clone(),
+            request.data,
+            None, // write_store_wrapper not used for namespace connections
+            request.write_options.lance_write_params,
+            self.read_consistency_interval,
+            self.server_side_query_enabled,
+            self.session.clone(),
+        )
+        .await?;
+
+        Ok(Arc::new(native_table))
     }
 
     async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
-        // Extract user-provided storage options from request
-        let user_storage_options = request
-            .lance_read_params
-            .as_ref()
-            .and_then(|lrp| lrp.store_options.as_ref())
-            .and_then(|so| so.storage_options.as_ref());
+        let native_table = NativeTable::open_from_namespace(
+            self.namespace.clone(),
+            &request.name,
+            request.namespace.clone(),
+            None, // write_store_wrapper not used for namespace connections
+            request.lance_read_params,
+            self.read_consistency_interval,
+            self.server_side_query_enabled,
+            self.session.clone(),
+        )
+        .await?;
 
-        let mut table_id = request.namespace.clone();
-        table_id.push(request.name.clone());
-
-        let describe_request = DescribeTableRequest {
-            id: Some(table_id.clone()),
-            version: None,
-        };
-        let response = self
-            .namespace
-            .describe_table(describe_request)
-            .await
-            .map_err(|e| Error::Runtime {
-                message: format!("Failed to describe table: {}", e),
-            })?;
-
-        let location = response.location.ok_or_else(|| Error::Runtime {
-            message: "Table location is missing from namespace response".to_string(),
-        })?;
-
-        let listing_db = self
-            .create_listing_database(
-                &location,
-                table_id.clone(),
-                user_storage_options,
-                response.storage_options.as_ref(),
-            )
-            .await?;
-
-        let namespace_client = self
-            .server_side_query_enabled
-            .then(|| self.namespace.clone());
-
-        let open_request = OpenTableRequest {
-            name: request.name.clone(),
-            namespace: request.namespace.clone(),
-            index_cache_size: request.index_cache_size,
-            lance_read_params: request.lance_read_params,
-            location: Some(location),
-            namespace_client,
-        };
-
-        listing_db.open_table(open_request).await
+        Ok(Arc::new(native_table))
     }
 
     async fn clone_table(&self, _request: CloneTableRequest) -> Result<Arc<dyn BaseTable>> {
@@ -394,7 +308,10 @@ impl Database for LanceNamespaceDatabase {
         let mut table_id = namespace.to_vec();
         table_id.push(name.to_string());
 
-        let drop_request = DropTableRequest { id: Some(table_id) };
+        let drop_request = DropTableRequest {
+            id: Some(table_id),
+            ..Default::default()
+        };
         self.namespace
             .drop_table(drop_request)
             .await
@@ -549,8 +466,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");
@@ -610,8 +526,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");
@@ -674,8 +589,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");
@@ -758,8 +672,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");
@@ -814,8 +727,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");
@@ -895,8 +807,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");
@@ -929,8 +840,7 @@ mod tests {
         // Create a child namespace first
         conn.create_namespace(CreateNamespaceRequest {
             id: Some(vec!["test_ns".into()]),
-            mode: None,
-            properties: None,
+            ..Default::default()
         })
         .await
         .expect("Failed to create namespace");

rust/lancedb/src/dataloader/permutation/shuffle.rs

@@ -171,7 +171,7 @@ impl Shuffler {
             // This is kind of an annoying limitation but if we allow runt clumps from batches then
             // clumps will get unaligned and we will mess up the clumps when we do the in-memory
             // shuffle step.  If this is a problem we can probably figure out a better way to do this.
-            if !is_last && batch.num_rows() as u64 % clump_size != 0 {
+            if !is_last && !(batch.num_rows() as u64).is_multiple_of(clump_size) {
                 return Err(Error::Runtime {
                     message: format!(
                         "Expected batch size ({}) to be divisible by clump size ({})",

rust/lancedb/src/dataloader/permutation/split.rs

@@ -1,12 +1,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::{
-    iter,
-    sync::{
-        atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
-        Arc,
-    },
+use std::sync::{
+    atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
+    Arc,
 };
 
 use arrow_array::{Array, BooleanArray, RecordBatch, UInt64Array};
@@ -158,7 +155,7 @@ impl Splitter {
                 remaining_in_split
             };
 
-            split_ids.extend(iter::repeat(split_id as u64).take(rows_to_add as usize));
+            split_ids.extend(std::iter::repeat_n(split_id as u64, rows_to_add as usize));
             if done {
                 // Quit early if we've run out of splits
                 break;
@@ -662,7 +659,7 @@ mod tests {
         assert_eq!(split_batch.num_rows(), total_split_sizes as usize);
         let mut expected = Vec::with_capacity(total_split_sizes as usize);
         for (i, size) in expected_split_sizes.iter().enumerate() {
-            expected.extend(iter::repeat(i as u64).take(*size as usize));
+            expected.extend(std::iter::repeat_n(i as u64, *size as usize));
         }
         let expected = Arc::new(UInt64Array::from(expected)) as Arc<dyn Array>;
 

rust/lancedb/src/embeddings.rs

@@ -120,8 +120,13 @@ impl MemoryRegistry {
 }
 
 /// A record batch reader that has embeddings applied to it
-/// This is a wrapper around another record batch reader that applies an embedding function
-/// when reading from the record batch
+///
+/// This is a wrapper around another record batch reader that applies embedding functions
+/// when reading from the record batch.
+///
+/// When multiple embedding functions are defined, they are computed in parallel using
+/// scoped threads to improve performance. For a single embedding function, computation
+/// is done inline without threading overhead.
 pub struct WithEmbeddings<R: RecordBatchReader> {
     inner: R,
     embeddings: Vec<(EmbeddingDefinition, Arc<dyn EmbeddingFunction>)>,
@@ -235,6 +240,48 @@ impl<R: RecordBatchReader> WithEmbeddings<R> {
             column_definitions,
         })
     }
+
+    fn compute_embeddings_parallel(&self, batch: &RecordBatch) -> Result<Vec<Arc<dyn Array>>> {
+        if self.embeddings.len() == 1 {
+            let (fld, func) = &self.embeddings[0];
+            let src_column =
+                batch
+                    .column_by_name(&fld.source_column)
+                    .ok_or_else(|| Error::InvalidInput {
+                        message: format!("Source column '{}' not found", fld.source_column),
+                    })?;
+            return Ok(vec![func.compute_source_embeddings(src_column.clone())?]);
+        }
+
+        // Parallel path: multiple embeddings
+        std::thread::scope(|s| {
+            let handles: Vec<_> = self
+                .embeddings
+                .iter()
+                .map(|(fld, func)| {
+                    let src_column = batch.column_by_name(&fld.source_column).ok_or_else(|| {
+                        Error::InvalidInput {
+                            message: format!("Source column '{}' not found", fld.source_column),
+                        }
+                    })?;
+
+                    let handle =
+                        s.spawn(move || func.compute_source_embeddings(src_column.clone()));
+
+                    Ok(handle)
+                })
+                .collect::<Result<_>>()?;
+
+            handles
+                .into_iter()
+                .map(|h| {
+                    h.join().map_err(|e| Error::Runtime {
+                        message: format!("Thread panicked during embedding computation: {:?}", e),
+                    })?
+                })
+                .collect()
+        })
+    }
 }
 
 impl<R: RecordBatchReader> Iterator for MaybeEmbedded<R> {
@@ -262,19 +309,19 @@ impl<R: RecordBatchReader> Iterator for WithEmbeddings<R> {
     fn next(&mut self) -> Option<Self::Item> {
         let batch = self.inner.next()?;
         match batch {
-            Ok(mut batch) => {
-                // todo: parallelize this
-                for (fld, func) in self.embeddings.iter() {
-                    let src_column = batch.column_by_name(&fld.source_column).unwrap();
-                    let embedding = match func.compute_source_embeddings(src_column.clone()) {
-                        Ok(embedding) => embedding,
-                        Err(e) => {
-                            return Some(Err(arrow_schema::ArrowError::ComputeError(format!(
-                                "Error computing embedding: {}",
-                                e
-                            ))))
-                        }
-                    };
+            Ok(batch) => {
+                let embeddings = match self.compute_embeddings_parallel(&batch) {
+                    Ok(emb) => emb,
+                    Err(e) => {
+                        return Some(Err(arrow_schema::ArrowError::ComputeError(format!(
+                            "Error computing embedding: {}",
+                            e
+                        ))))
+                    }
+                };
+
+                let mut batch = batch;
+                for ((fld, _), embedding) in self.embeddings.iter().zip(embeddings.iter()) {
                     let dst_field_name = fld
                         .dest_column
                         .clone()
@@ -286,7 +333,7 @@ impl<R: RecordBatchReader> Iterator for WithEmbeddings<R> {
                         embedding.nulls().is_some(),
                     );
 
-                    match batch.try_with_column(dst_field.clone(), embedding) {
+                    match batch.try_with_column(dst_field.clone(), embedding.clone()) {
                         Ok(b) => batch = b,
                         Err(e) => return Some(Err(e)),
                     };

rust/lancedb/src/index/vector.rs

@@ -297,10 +297,10 @@ impl IvfPqIndexBuilder {
 }
 
 pub(crate) fn suggested_num_sub_vectors(dim: u32) -> u32 {
-    if dim % 16 == 0 {
+    if dim.is_multiple_of(16) {
         // Should be more aggressive than this default.
         dim / 16
-    } else if dim % 8 == 0 {
+    } else if dim.is_multiple_of(8) {
         dim / 8
     } else {
         log::warn!(

rust/lancedb/src/lib.rs

@@ -25,13 +25,14 @@
 //!
 //! ## Crate Features
 //!
-//! ### Experimental Features
-//!
-//! These features are not enabled by default.  They are experimental or in-development features that
-//! are not yet ready to be released.
-//!
-//! - `remote` - Enable remote client to connect to LanceDB cloud.  This is not yet fully implemented
-//!   and should not be enabled.
+//! - `aws` - Enable AWS S3 object store support.
+//! - `dynamodb` - Enable DynamoDB manifest store support.
+//! - `azure` - Enable Azure Blob Storage object store support.
+//! - `gcs` - Enable Google Cloud Storage object store support.
+//! - `oss` - Enable Alibaba Cloud OSS object store support.
+//! - `remote` - Enable remote client to connect to LanceDB cloud.
+//! - `huggingface` - Enable HuggingFace Hub integration for loading datasets from the Hub.
+//! - `fp16kernels` - Enable FP16 kernels for faster vector search on CPU.
 //!
 //! ### Quick Start
 //!
@@ -50,17 +51,15 @@
 //! - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud object store
 //! - `db://dbname` - Lance Cloud
 //!
-//! You can also use [`ConnectOptions`] to configure the connection to the database.
+//! You can also use [`ConnectBuilder`] to configure the connection to the database.
 //!
 //! ```rust
-//! use object_store::aws::AwsCredential;
 //! # tokio::runtime::Runtime::new().unwrap().block_on(async {
 //! let db = lancedb::connect("data/sample-lancedb")
-//!     .aws_creds(AwsCredential {
-//!         key_id: "some_key".to_string(),
-//!         secret_key: "some_secret".to_string(),
-//!         token: None,
-//!     })
+//!     .storage_options([
+//!         ("aws_access_key_id", "some_key"),
+//!         ("aws_secret_access_key", "some_secret"),
+//!     ])
 //!     .execute()
 //!     .await
 //!     .unwrap();

rust/lancedb/src/remote/db.rs

@@ -1718,8 +1718,7 @@ mod tests {
             let namespace = vec!["test_ns".to_string()];
             conn.create_namespace(CreateNamespaceRequest {
                 id: Some(namespace.clone()),
-                mode: None,
-                properties: None,
+                ..Default::default()
             })
             .await
             .expect("Failed to create namespace");
@@ -1744,8 +1743,7 @@ mod tests {
             let list_response = conn
                 .list_tables(ListTablesRequest {
                     id: Some(namespace.clone()),
-                    page_token: None,
-                    limit: None,
+                    ..Default::default()
                 })
                 .await
                 .expect("Failed to list tables");
@@ -1756,8 +1754,7 @@ mod tests {
             let list_response = namespace_client
                 .list_tables(ListTablesRequest {
                     id: Some(namespace.clone()),
-                    page_token: None,
-                    limit: None,
+                    ..Default::default()
                 })
                 .await
                 .unwrap();
@@ -1797,8 +1794,7 @@ mod tests {
             let namespace = vec!["multi_table_ns".to_string()];
             conn.create_namespace(CreateNamespaceRequest {
                 id: Some(namespace.clone()),
-                mode: None,
-                properties: None,
+                ..Default::default()
             })
             .await
             .expect("Failed to create namespace");
@@ -1823,8 +1819,7 @@ mod tests {
             let list_response = conn
                 .list_tables(ListTablesRequest {
                     id: Some(namespace.clone()),
-                    page_token: None,
-                    limit: None,
+                    ..Default::default()
                 })
                 .await
                 .unwrap();

rust/lancedb/src/remote/table.rs

@@ -204,6 +204,7 @@ pub struct RemoteTable<S: HttpSend = Sender> {
     server_version: ServerVersion,
 
     version: RwLock<Option<u64>>,
+    location: RwLock<Option<String>>,
 }
 
 impl<S: HttpSend> RemoteTable<S> {
@@ -221,6 +222,7 @@ impl<S: HttpSend> RemoteTable<S> {
             identifier,
             server_version,
             version: RwLock::new(None),
+            location: RwLock::new(None),
         }
     }
 
@@ -639,6 +641,7 @@ impl<S: HttpSend> RemoteTable<S> {
 struct TableDescription {
     version: u64,
     schema: JsonSchema,
+    location: Option<String>,
 }
 
 impl<S: HttpSend> std::fmt::Display for RemoteTable<S> {
@@ -667,6 +670,7 @@ mod test_utils {
                 identifier: name,
                 server_version: version.map(ServerVersion).unwrap_or_default(),
                 version: RwLock::new(None),
+                location: RwLock::new(None),
             }
         }
     }
@@ -1088,6 +1092,17 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
                     body["num_partitions"] = serde_json::Value::Number(num_partitions.into());
                 }
             }
+            Index::IvfRq(index) => {
+                body[INDEX_TYPE_KEY] = serde_json::Value::String("IVF_RQ".to_string());
+                body[METRIC_TYPE_KEY] =
+                    serde_json::Value::String(index.distance_type.to_string().to_lowercase());
+                if let Some(num_partitions) = index.num_partitions {
+                    body["num_partitions"] = serde_json::Value::Number(num_partitions.into());
+                }
+                if let Some(num_bits) = index.num_bits {
+                    body["num_bits"] = serde_json::Value::Number(num_bits.into());
+                }
+            }
             Index::BTree(_) => {
                 body[INDEX_TYPE_KEY] = serde_json::Value::String("BTREE".to_string());
             }
@@ -1450,14 +1465,42 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
             message: "table_definition is not supported on LanceDB cloud.".into(),
         })
     }
-    fn dataset_uri(&self) -> &str {
-        "NOT_SUPPORTED"
+    async fn uri(&self) -> Result<String> {
+        // Check if we already have the location cached
+        {
+            let location = self.location.read().await;
+            if let Some(ref loc) = *location {
+                return Ok(loc.clone());
+            }
+        }
+
+        // Fetch from server via describe
+        let description = self.describe().await?;
+        let location = description.location.ok_or_else(|| Error::NotSupported {
+            message: "Table URI not supported by the server".into(),
+        })?;
+
+        // Cache the location for future use
+        {
+            let mut cached_location = self.location.write().await;
+            *cached_location = Some(location.clone());
+        }
+
+        Ok(location)
     }
 
     async fn storage_options(&self) -> Option<HashMap<String, String>> {
         None
     }
 
+    async fn initial_storage_options(&self) -> Option<HashMap<String, String>> {
+        None
+    }
+
+    async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        Ok(None)
+    }
+
     async fn stats(&self) -> Result<TableStatistics> {
         let request = self
             .client
@@ -3321,4 +3364,69 @@ mod tests {
         let result = table.drop_columns(&["old_col1", "old_col2"]).await.unwrap();
         assert_eq!(result.version, 5);
     }
+
+    #[tokio::test]
+    async fn test_uri() {
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.method(), "POST");
+            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
+
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"version": 1, "schema": {"fields": []}, "location": "s3://bucket/path/to/table"}"#)
+                .unwrap()
+        });
+
+        let uri = table.uri().await.unwrap();
+        assert_eq!(uri, "s3://bucket/path/to/table");
+    }
+
+    #[tokio::test]
+    async fn test_uri_missing_location() {
+        let table = Table::new_with_handler("my_table", |request| {
+            assert_eq!(request.method(), "POST");
+            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
+
+            // Server returns response without location field
+            http::Response::builder()
+                .status(200)
+                .body(r#"{"version": 1, "schema": {"fields": []}}"#)
+                .unwrap()
+        });
+
+        let result = table.uri().await;
+        assert!(result.is_err());
+        assert!(matches!(&result, Err(Error::NotSupported { .. })));
+    }
+
+    #[tokio::test]
+    async fn test_uri_caching() {
+        use std::sync::atomic::{AtomicUsize, Ordering};
+        use std::sync::Arc;
+
+        let call_count = Arc::new(AtomicUsize::new(0));
+        let call_count_clone = call_count.clone();
+
+        let table = Table::new_with_handler("my_table", move |request| {
+            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
+            call_count_clone.fetch_add(1, Ordering::SeqCst);
+
+            http::Response::builder()
+                .status(200)
+                .body(
+                    r#"{"version": 1, "schema": {"fields": []}, "location": "gs://bucket/table"}"#,
+                )
+                .unwrap()
+        });
+
+        // First call should fetch from server
+        let uri1 = table.uri().await.unwrap();
+        assert_eq!(uri1, "gs://bucket/table");
+        assert_eq!(call_count.load(Ordering::SeqCst), 1);
+
+        // Second call should use cached value
+        let uri2 = table.uri().await.unwrap();
+        assert_eq!(uri2, "gs://bucket/table");
+        assert_eq!(call_count.load(Ordering::SeqCst), 1); // Still 1, no new call
+    }
 }

rust/lancedb/src/table.rs

@@ -29,7 +29,7 @@ use lance::dataset::{
 use lance::dataset::{MergeInsertBuilder as LanceMergeInsertBuilder, WhenNotMatchedBySource};
 use lance::index::vector::utils::infer_vector_dim;
 use lance::index::vector::VectorIndexParams;
-use lance::io::WrappingObjectStore;
+use lance::io::{ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::exec::{analyze_plan as lance_analyze_plan, execute_plan};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams};
@@ -40,9 +40,10 @@ use lance_index::vector::pq::PQBuildParams;
 use lance_index::vector::sq::builder::SQBuildParams;
 use lance_index::DatasetIndexExt;
 use lance_index::IndexType;
+use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsAccessor};
 use lance_namespace::models::{
-    QueryTableRequest as NsQueryTableRequest, QueryTableRequestFullTextQuery,
-    QueryTableRequestVector, StringFtsQuery,
+    QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns,
+    QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery,
 };
 use lance_namespace::LanceNamespace;
 use lance_table::format::Manifest;
@@ -607,10 +608,20 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     async fn list_versions(&self) -> Result<Vec<Version>>;
     /// Get the table definition.
     async fn table_definition(&self) -> Result<TableDefinition>;
-    /// Get the table URI
-    fn dataset_uri(&self) -> &str;
+    /// Get the table URI (storage location)
+    async fn uri(&self) -> Result<String>;
     /// Get the storage options used when opening this table, if any.
+    #[deprecated(since = "0.25.0", note = "Use initial_storage_options() instead")]
     async fn storage_options(&self) -> Option<HashMap<String, String>>;
+    /// Get the initial storage options that were passed in when opening this table.
+    ///
+    /// For dynamically refreshed options (e.g., credential vending), use [`Self::latest_storage_options`].
+    async fn initial_storage_options(&self) -> Option<HashMap<String, String>>;
+    /// Get the latest storage options, refreshing from provider if configured.
+    ///
+    /// Returns `Ok(Some(options))` if storage options are available (static or refreshed),
+    /// `Ok(None)` if no storage options were configured, or `Err(...)` if refresh failed.
+    async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>>;
     /// Poll until the columns are fully indexed. Will return Error::Timeout if the columns
     /// are not fully indexed within the timeout.
     async fn wait_for_index(
@@ -1316,20 +1327,43 @@ impl Table {
         self.inner.list_indices().await
     }
 
-    /// Get the underlying dataset URI
+    /// Get the table URI (storage location)
     ///
-    /// Warning: This is an internal API and the return value is subject to change.
-    pub fn dataset_uri(&self) -> &str {
-        self.inner.dataset_uri()
+    /// Returns the full storage location of the table (e.g., S3/GCS path).
+    /// For remote tables, this fetches the location from the server via describe.
+    pub async fn uri(&self) -> Result<String> {
+        self.inner.uri().await
     }
 
     /// Get the storage options used when opening this table, if any.
     ///
     /// Warning: This is an internal API and the return value is subject to change.
+    #[deprecated(since = "0.25.0", note = "Use initial_storage_options() instead")]
     pub async fn storage_options(&self) -> Option<HashMap<String, String>> {
+        #[allow(deprecated)]
         self.inner.storage_options().await
     }
 
+    /// Get the initial storage options that were passed in when opening this table.
+    ///
+    /// For dynamically refreshed options (e.g., credential vending), use [`Self::latest_storage_options`].
+    ///
+    /// Warning: This is an internal API and the return value is subject to change.
+    pub async fn initial_storage_options(&self) -> Option<HashMap<String, String>> {
+        self.inner.initial_storage_options().await
+    }
+
+    /// Get the latest storage options, refreshing from provider if configured.
+    ///
+    /// This method is useful for credential vending scenarios where storage options
+    /// may be refreshed dynamically. If no dynamic provider is configured, this
+    /// returns the initial static options.
+    ///
+    /// Warning: This is an internal API and the return value is subject to change.
+    pub async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        self.inner.latest_storage_options().await
+    }
+
     /// Get statistics about an index.
     /// Returns None if the index does not exist.
     pub async fn index_stats(
@@ -1611,6 +1645,101 @@ impl NativeTable {
         self
     }
 
+    /// Opens an existing Table using a namespace client.
+    ///
+    /// This method uses `DatasetBuilder::from_namespace` to open the table, which
+    /// automatically fetches the table location and storage options from the namespace.
+    /// This eliminates the need to pre-fetch and merge storage options before opening.
+    ///
+    /// # Arguments
+    ///
+    /// * `namespace_client` - The namespace client to use for fetching table metadata
+    /// * `name` - The table name
+    /// * `namespace` - The namespace path (e.g., vec!["parent", "child"])
+    /// * `write_store_wrapper` - Optional wrapper for the object store on write path
+    /// * `params` - Optional read parameters
+    /// * `read_consistency_interval` - Optional interval for read consistency
+    /// * `server_side_query_enabled` - Whether to enable server-side query execution.
+    ///   When true, the namespace_client will be stored and queries will be executed
+    ///   on the namespace server. When false, the namespace is only used for opening
+    ///   the table, and queries are executed locally.
+    /// * `session` - Optional session for object stores and caching
+    ///
+    /// # Returns
+    ///
+    /// * A [NativeTable] object.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn open_from_namespace(
+        namespace_client: Arc<dyn LanceNamespace>,
+        name: &str,
+        namespace: Vec<String>,
+        write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
+        params: Option<ReadParams>,
+        read_consistency_interval: Option<std::time::Duration>,
+        server_side_query_enabled: bool,
+        session: Option<Arc<lance::session::Session>>,
+    ) -> Result<Self> {
+        let mut params = params.unwrap_or_default();
+
+        // Set the session in read params
+        if let Some(sess) = session {
+            params.session(sess);
+        }
+
+        // patch the params if we have a write store wrapper
+        let params = match write_store_wrapper.clone() {
+            Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
+            None => params,
+        };
+
+        // Build table_id from namespace + name
+        let mut table_id = namespace.clone();
+        table_id.push(name.to_string());
+
+        // Use DatasetBuilder::from_namespace which automatically fetches location
+        // and storage options from the namespace
+        let builder = DatasetBuilder::from_namespace(namespace_client.clone(), table_id)
+            .await
+            .map_err(|e| match e {
+                lance::Error::Namespace { source, .. } => Error::Runtime {
+                    message: format!("Failed to get table info from namespace: {:?}", source),
+                },
+                source => Error::Lance { source },
+            })?;
+
+        let dataset = builder
+            .with_read_params(params)
+            .load()
+            .await
+            .map_err(|e| match e {
+                lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
+                    name: name.to_string(),
+                    source: Box::new(e),
+                },
+                source => Error::Lance { source },
+            })?;
+
+        let uri = dataset.uri().to_string();
+        let dataset = DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval);
+        let id = Self::build_id(&namespace, name);
+
+        let stored_namespace_client = if server_side_query_enabled {
+            Some(namespace_client)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            name: name.to_string(),
+            namespace,
+            id,
+            uri,
+            dataset,
+            read_consistency_interval,
+            namespace_client: stored_namespace_client,
+        })
+    }
+
     fn get_table_name(uri: &str) -> Result<String> {
         let path = Path::new(uri);
         let name = path
@@ -1722,6 +1851,108 @@ impl NativeTable {
         .await
     }
 
+    /// Creates a new Table using a namespace client for storage options.
+    ///
+    /// This method sets up a `StorageOptionsProvider` from the namespace client,
+    /// enabling automatic credential refresh for cloud storage. The namespace
+    /// is used for:
+    /// 1. Setting up storage options provider for credential vending
+    /// 2. Optionally enabling server-side query execution
+    ///
+    /// # Arguments
+    ///
+    /// * `namespace_client` - The namespace client to use for storage options
+    /// * `uri` - The URI to the table (obtained from create_empty_table response)
+    /// * `name` - The table name
+    /// * `namespace` - The namespace path (e.g., vec!["parent", "child"])
+    /// * `batches` - RecordBatch to be saved in the database
+    /// * `write_store_wrapper` - Optional wrapper for the object store on write path
+    /// * `params` - Optional write parameters
+    /// * `read_consistency_interval` - Optional interval for read consistency
+    /// * `server_side_query_enabled` - Whether to enable server-side query execution
+    ///
+    /// # Returns
+    ///
+    /// * A [NativeTable] object.
+    #[allow(clippy::too_many_arguments)]
+    pub async fn create_from_namespace(
+        namespace_client: Arc<dyn LanceNamespace>,
+        uri: &str,
+        name: &str,
+        namespace: Vec<String>,
+        batches: impl StreamingWriteSource,
+        write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
+        params: Option<WriteParams>,
+        read_consistency_interval: Option<std::time::Duration>,
+        server_side_query_enabled: bool,
+        session: Option<Arc<lance::session::Session>>,
+    ) -> Result<Self> {
+        // Build table_id from namespace + name for the storage options provider
+        let mut table_id = namespace.clone();
+        table_id.push(name.to_string());
+
+        // Set up storage options provider from namespace
+        let storage_options_provider = Arc::new(LanceNamespaceStorageOptionsProvider::new(
+            namespace_client.clone(),
+            table_id,
+        ));
+
+        // Start with provided params or defaults
+        let mut params = params.unwrap_or_default();
+
+        // Set the session in write params
+        if let Some(sess) = session {
+            params.session = Some(sess);
+        }
+
+        // Ensure store_params exists and set the storage options provider
+        let store_params = params
+            .store_params
+            .get_or_insert_with(ObjectStoreParams::default);
+        let accessor = match store_params.storage_options().cloned() {
+            Some(options) => {
+                StorageOptionsAccessor::with_initial_and_provider(options, storage_options_provider)
+            }
+            None => StorageOptionsAccessor::with_provider(storage_options_provider),
+        };
+        store_params.storage_options_accessor = Some(Arc::new(accessor));
+
+        // Patch the params if we have a write store wrapper
+        let params = match write_store_wrapper.clone() {
+            Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
+            None => params,
+        };
+
+        let insert_builder = InsertBuilder::new(uri).with_params(&params);
+        let dataset = insert_builder
+            .execute_stream(batches)
+            .await
+            .map_err(|e| match e {
+                lance::Error::DatasetAlreadyExists { .. } => Error::TableAlreadyExists {
+                    name: name.to_string(),
+                },
+                source => Error::Lance { source },
+            })?;
+
+        let id = Self::build_id(&namespace, name);
+
+        let stored_namespace_client = if server_side_query_enabled {
+            Some(namespace_client)
+        } else {
+            None
+        };
+
+        Ok(Self {
+            name: name.to_string(),
+            namespace,
+            id,
+            uri: uri.to_string(),
+            dataset: DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval),
+            read_consistency_interval,
+            namespace_client: stored_namespace_client,
+        })
+    }
+
     async fn optimize_indices(&self, options: &OptimizeOptions) -> Result<()> {
         info!("LanceDB: optimizing indices: {:?}", options);
         self.dataset
@@ -1860,7 +2091,7 @@ impl NativeTable {
             return provided;
         }
         let suggested = suggested_num_sub_vectors(dim);
-        if num_bits.is_some_and(|num_bits| num_bits == 4) && suggested % 2 != 0 {
+        if num_bits.is_some_and(|num_bits| num_bits == 4) && !suggested.is_multiple_of(2) {
             // num_sub_vectors must be even when 4 bits are used
             suggested + 1
         } else {
@@ -2152,7 +2383,10 @@ impl NativeTable {
                 // Convert select to columns list
                 let columns = match &vq.base.select {
                     Select::All => None,
-                    Select::Columns(cols) => Some(cols.clone()),
+                    Select::Columns(cols) => Some(Box::new(QueryTableRequestColumns {
+                        column_names: Some(cols.clone()),
+                        column_aliases: None,
+                    })),
                     Select::Dynamic(_) => {
                         return Err(Error::NotSupported {
                             message:
@@ -2205,7 +2439,7 @@ impl NativeTable {
                     with_row_id: Some(vq.base.with_row_id),
                     bypass_vector_index: Some(!vq.use_index),
                     full_text_query,
-                    version: None,
+                    ..Default::default()
                 })
             }
             AnyQuery::Query(q) => {
@@ -2225,7 +2459,10 @@ impl NativeTable {
 
                 let columns = match &q.select {
                     Select::All => None,
-                    Select::Columns(cols) => Some(cols.clone()),
+                    Select::Columns(cols) => Some(Box::new(QueryTableRequestColumns {
+                        column_names: Some(cols.clone()),
+                        column_aliases: None,
+                    })),
                     Select::Dynamic(_) => {
                         return Err(Error::NotSupported {
                             message: "Dynamic columns are not supported for server-side query"
@@ -2264,18 +2501,11 @@ impl NativeTable {
                     columns,
                     prefilter: Some(q.prefilter),
                     offset: q.offset.map(|o| o as i32),
-                    ef: None,
-                    refine_factor: None,
-                    distance_type: None,
-                    nprobes: None,
                     vector_column: None, // No vector column for plain queries
                     with_row_id: Some(q.with_row_id),
                     bypass_vector_index: Some(true), // No vector index for plain queries
                     full_text_query,
-                    version: None,
-                    fast_search: None,
-                    lower_bound: None,
-                    upper_bound: None,
+                    ..Default::default()
                 })
             }
         }
@@ -3028,16 +3258,25 @@ impl BaseTable for NativeTable {
         Ok(results.into_iter().flatten().collect())
     }
 
-    fn dataset_uri(&self) -> &str {
-        self.uri.as_str()
+    async fn uri(&self) -> Result<String> {
+        Ok(self.uri.clone())
     }
 
     async fn storage_options(&self) -> Option<HashMap<String, String>> {
+        self.initial_storage_options().await
+    }
+
+    async fn initial_storage_options(&self) -> Option<HashMap<String, String>> {
         self.dataset
             .get()
             .await
             .ok()
-            .and_then(|dataset| dataset.storage_options().cloned())
+            .and_then(|dataset| dataset.initial_storage_options().cloned())
+    }
+
+    async fn latest_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        let dataset = self.dataset.get().await?;
+        Ok(dataset.latest_storage_options().await?.map(|o| o.0))
     }
 
     async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {
@@ -3202,7 +3441,6 @@ pub struct FragmentSummaryStats {
 #[cfg(test)]
 #[allow(deprecated)]
 mod tests {
-    use std::iter;
     use std::sync::atomic::{AtomicBool, Ordering};
     use std::sync::Arc;
     use std::time::Duration;
@@ -3819,7 +4057,7 @@ mod tests {
                 schema.clone(),
                 vec![
                     Arc::new(Int32Array::from_iter_values(offset..(offset + 10))),
-                    Arc::new(Int32Array::from_iter_values(iter::repeat(age).take(10))),
+                    Arc::new(Int32Array::from_iter_values(std::iter::repeat_n(age, 10))),
                 ],
             )],
             schema,
@@ -4950,7 +5188,13 @@ mod tests {
         assert_eq!(ns_request.k, 10);
         assert_eq!(ns_request.offset, Some(5));
         assert_eq!(ns_request.filter, Some("id > 0".to_string()));
-        assert_eq!(ns_request.columns, Some(vec!["id".to_string()]));
+        assert_eq!(
+            ns_request
+                .columns
+                .as_ref()
+                .and_then(|c| c.column_names.as_ref()),
+            Some(&vec!["id".to_string()])
+        );
         assert_eq!(ns_request.vector_column, Some("vector".to_string()));
         assert_eq!(ns_request.distance_type, Some("l2".to_string()));
         assert!(ns_request.vector.single_vector.is_some());
@@ -4991,7 +5235,13 @@ mod tests {
         assert_eq!(ns_request.k, 20);
         assert_eq!(ns_request.offset, Some(5));
         assert_eq!(ns_request.filter, Some("id > 5".to_string()));
-        assert_eq!(ns_request.columns, Some(vec!["id".to_string()]));
+        assert_eq!(
+            ns_request
+                .columns
+                .as_ref()
+                .and_then(|c| c.column_names.as_ref()),
+            Some(&vec!["id".to_string()])
+        );
         assert_eq!(ns_request.with_row_id, Some(true));
         assert_eq!(ns_request.bypass_vector_index, Some(true));
         assert!(ns_request.vector_column.is_none()); // No vector column for plain queries

rust/lancedb/src/test_utils/connection.rs

@@ -5,16 +5,19 @@
 
 use regex::Regex;
 use std::env;
-use std::io::{BufRead, BufReader};
-use std::process::{Child, ChildStdout, Command, Stdio};
+use std::process::Stdio;
+use tokio::io::{AsyncBufReadExt, BufReader};
+use tokio::process::{Child, ChildStdout, Command};
+use tokio::sync::mpsc;
 
 use crate::{connect, Connection};
-use anyhow::{bail, Result};
+use anyhow::{anyhow, bail, Result};
 use tempfile::{tempdir, TempDir};
 
 pub struct TestConnection {
     pub uri: String,
     pub connection: Connection,
+    pub is_remote: bool,
     _temp_dir: Option<TempDir>,
     _process: Option<TestProcess>,
 }
@@ -37,6 +40,56 @@ pub async fn new_test_connection() -> Result<TestConnection> {
     }
 }
 
+async fn spawn_stdout_reader(
+    mut stdout: BufReader<ChildStdout>,
+    port_sender: mpsc::Sender<anyhow::Result<String>>,
+) -> tokio::task::JoinHandle<()> {
+    let print_stdout = env::var("PRINT_LANCEDB_TEST_CONNECTION_SCRIPT_OUTPUT").is_ok();
+    tokio::spawn(async move {
+        let mut line = String::new();
+        let re = Regex::new(r"Query node now listening on 0.0.0.0:(.*)").unwrap();
+        loop {
+            line.clear();
+            let result = stdout.read_line(&mut line).await;
+            if let Err(err) = result {
+                port_sender
+                    .send(Err(anyhow!(
+                        "error while reading from process output: {}",
+                        err
+                    )))
+                    .await
+                    .unwrap();
+                return;
+            } else if result.unwrap() == 0 {
+                port_sender
+                    .send(Err(anyhow!(
+                        " hit EOF before reading port from process output."
+                    )))
+                    .await
+                    .unwrap();
+                return;
+            }
+            if re.is_match(&line) {
+                let caps = re.captures(&line).unwrap();
+                port_sender.send(Ok(caps[1].to_string())).await.unwrap();
+                break;
+            }
+        }
+        loop {
+            line.clear();
+            match stdout.read_line(&mut line).await {
+                Err(_) => return,
+                Ok(0) => return,
+                Ok(_size) => {
+                    if print_stdout {
+                        print!("{}", line);
+                    }
+                }
+            }
+        }
+    })
+}
+
 async fn new_remote_connection(script_path: &str) -> Result<TestConnection> {
     let temp_dir = tempdir()?;
     let data_path = temp_dir.path().to_str().unwrap().to_string();
@@ -57,38 +110,25 @@ async fn new_remote_connection(script_path: &str) -> Result<TestConnection> {
         child: child_result.unwrap(),
     };
     let stdout = BufReader::new(process.child.stdout.take().unwrap());
-    let port = read_process_port(stdout)?;
+    let (port_sender, mut port_receiver) = mpsc::channel(5);
+    let _reader = spawn_stdout_reader(stdout, port_sender).await;
+    let port = match port_receiver.recv().await {
+        None => bail!("Unable to determine the port number used by the phalanx process we spawned, because the reader thread was closed too soon."),
+        Some(Err(err)) => bail!("Unable to determine the port number used by the phalanx process we spawned, because of an error, {}", err),
+        Some(Ok(port)) => port,
+    };
     let uri = "db://test";
     let host_override = format!("http://localhost:{}", port);
     let connection = create_new_connection(uri, &host_override).await?;
     Ok(TestConnection {
         uri: uri.to_string(),
         connection,
+        is_remote: true,
         _temp_dir: Some(temp_dir),
         _process: Some(process),
     })
 }
 
-fn read_process_port(mut stdout: BufReader<ChildStdout>) -> Result<String> {
-    let mut line = String::new();
-    let re = Regex::new(r"Query node now listening on 0.0.0.0:(.*)").unwrap();
-    loop {
-        let result = stdout.read_line(&mut line);
-        if let Err(err) = result {
-            bail!(format!(
-                "read_process_port: error while reading from process output: {}",
-                err
-            ));
-        } else if result.unwrap() == 0 {
-            bail!("read_process_port: hit EOF before reading port from process output.");
-        }
-        if re.is_match(&line) {
-            let caps = re.captures(&line).unwrap();
-            return Ok(caps[1].to_string());
-        }
-    }
-}
-
 #[cfg(feature = "remote")]
 async fn create_new_connection(uri: &str, host_override: &str) -> crate::error::Result<Connection> {
     connect(uri)
@@ -114,6 +154,7 @@ async fn new_local_connection() -> Result<TestConnection> {
     Ok(TestConnection {
         uri: uri.to_string(),
         connection,
+        is_remote: false,
         _temp_dir: Some(temp_dir),
         _process: None,
     })

rust/lancedb/tests/embedding_registry_test.rs

@@ -4,7 +4,6 @@
 use std::{
     borrow::Cow,
     collections::{HashMap, HashSet},
-    iter::repeat,
     sync::Arc,
 };
 
@@ -268,9 +267,10 @@ fn create_some_records() -> Result<impl IntoArrow> {
             schema.clone(),
             vec![
                 Arc::new(Int32Array::from_iter_values(0..TOTAL as i32)),
-                Arc::new(StringArray::from_iter(
-                    repeat(Some("hello world".to_string())).take(TOTAL),
-                )),
+                Arc::new(StringArray::from_iter(std::iter::repeat_n(
+                    Some("hello world".to_string()),
+                    TOTAL,
+                ))),
             ],
         )
         .unwrap()]

rust/lancedb/tests/embeddings_parallel_test.rs

@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use std::{
+    borrow::Cow,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+use arrow::buffer::NullBuffer;
+use arrow_array::{
+    Array, FixedSizeListArray, Float32Array, RecordBatch, RecordBatchIterator, StringArray,
+};
+use arrow_schema::{DataType, Field, Schema};
+use lancedb::{
+    embeddings::{EmbeddingDefinition, EmbeddingFunction, MaybeEmbedded, WithEmbeddings},
+    Error, Result,
+};
+
+#[derive(Debug)]
+struct SlowMockEmbed {
+    name: String,
+    dim: usize,
+    delay_ms: u64,
+    call_count: Arc<AtomicUsize>,
+}
+
+impl SlowMockEmbed {
+    pub fn new(name: String, dim: usize, delay_ms: u64) -> Self {
+        Self {
+            name,
+            dim,
+            delay_ms,
+            call_count: Arc::new(AtomicUsize::new(0)),
+        }
+    }
+
+    pub fn get_call_count(&self) -> usize {
+        self.call_count.load(Ordering::SeqCst)
+    }
+}
+
+impl EmbeddingFunction for SlowMockEmbed {
+    fn name(&self) -> &str {
+        &self.name
+    }
+
+    fn source_type(&self) -> Result<Cow<'_, DataType>> {
+        Ok(Cow::Owned(DataType::Utf8))
+    }
+
+    fn dest_type(&self) -> Result<Cow<'_, DataType>> {
+        Ok(Cow::Owned(DataType::new_fixed_size_list(
+            DataType::Float32,
+            self.dim as _,
+            true,
+        )))
+    }
+
+    fn compute_source_embeddings(&self, source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        // Simulate slow embedding computation
+        std::thread::sleep(Duration::from_millis(self.delay_ms));
+        self.call_count.fetch_add(1, Ordering::SeqCst);
+
+        let len = source.len();
+        let inner = Arc::new(Float32Array::from(vec![Some(1.0); len * self.dim]));
+        let field = Field::new("item", inner.data_type().clone(), false);
+        let arr = FixedSizeListArray::new(
+            Arc::new(field),
+            self.dim as _,
+            inner,
+            Some(NullBuffer::new_valid(len)),
+        );
+
+        Ok(Arc::new(arr))
+    }
+
+    fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+        unimplemented!()
+    }
+}
+
+fn create_test_batch() -> Result<RecordBatch> {
+    let schema = Arc::new(Schema::new(vec![Field::new("text", DataType::Utf8, false)]));
+    let text = StringArray::from(vec!["hello", "world"]);
+    RecordBatch::try_new(schema, vec![Arc::new(text)]).map_err(|e| Error::Runtime {
+        message: format!("Failed to create test batch: {}", e),
+    })
+}
+
+#[test]
+fn test_single_embedding_fast_path() {
+    // Single embedding should execute without spawning threads
+    let batch = create_test_batch().unwrap();
+    let schema = batch.schema();
+
+    let embed = Arc::new(SlowMockEmbed::new("test".to_string(), 2, 10));
+    let embedding_def = EmbeddingDefinition::new("text", "test", Some("embedding"));
+
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+    let embeddings = vec![(embedding_def, embed.clone() as Arc<dyn EmbeddingFunction>)];
+    let mut with_embeddings = WithEmbeddings::new(reader, embeddings);
+
+    let result = with_embeddings.next().unwrap().unwrap();
+    assert!(result.column_by_name("embedding").is_some());
+    assert_eq!(embed.get_call_count(), 1);
+}
+
+#[test]
+fn test_multiple_embeddings_parallel() {
+    // Multiple embeddings should execute in parallel
+    let batch = create_test_batch().unwrap();
+    let schema = batch.schema();
+
+    let embed1 = Arc::new(SlowMockEmbed::new("embed1".to_string(), 2, 100));
+    let embed2 = Arc::new(SlowMockEmbed::new("embed2".to_string(), 3, 100));
+    let embed3 = Arc::new(SlowMockEmbed::new("embed3".to_string(), 4, 100));
+
+    let def1 = EmbeddingDefinition::new("text", "embed1", Some("emb1"));
+    let def2 = EmbeddingDefinition::new("text", "embed2", Some("emb2"));
+    let def3 = EmbeddingDefinition::new("text", "embed3", Some("emb3"));
+
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+    let embeddings = vec![
+        (def1, embed1.clone() as Arc<dyn EmbeddingFunction>),
+        (def2, embed2.clone() as Arc<dyn EmbeddingFunction>),
+        (def3, embed3.clone() as Arc<dyn EmbeddingFunction>),
+    ];
+    let mut with_embeddings = WithEmbeddings::new(reader, embeddings);
+
+    let result = with_embeddings.next().unwrap().unwrap();
+
+    // Verify all embedding columns are present
+    assert!(result.column_by_name("emb1").is_some());
+    assert!(result.column_by_name("emb2").is_some());
+    assert!(result.column_by_name("emb3").is_some());
+
+    // Verify all embeddings were computed
+    assert_eq!(embed1.get_call_count(), 1);
+    assert_eq!(embed2.get_call_count(), 1);
+    assert_eq!(embed3.get_call_count(), 1);
+}
+
+#[test]
+fn test_embedding_column_order_preserved() {
+    // Verify that embedding columns are added in the same order as definitions
+    let batch = create_test_batch().unwrap();
+    let schema = batch.schema();
+
+    let embed1 = Arc::new(SlowMockEmbed::new("embed1".to_string(), 2, 10));
+    let embed2 = Arc::new(SlowMockEmbed::new("embed2".to_string(), 3, 10));
+    let embed3 = Arc::new(SlowMockEmbed::new("embed3".to_string(), 4, 10));
+
+    let def1 = EmbeddingDefinition::new("text", "embed1", Some("first"));
+    let def2 = EmbeddingDefinition::new("text", "embed2", Some("second"));
+    let def3 = EmbeddingDefinition::new("text", "embed3", Some("third"));
+
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+    let embeddings = vec![
+        (def1, embed1 as Arc<dyn EmbeddingFunction>),
+        (def2, embed2 as Arc<dyn EmbeddingFunction>),
+        (def3, embed3 as Arc<dyn EmbeddingFunction>),
+    ];
+    let mut with_embeddings = WithEmbeddings::new(reader, embeddings);
+
+    let result = with_embeddings.next().unwrap().unwrap();
+    let result_schema = result.schema();
+
+    // Original column is first
+    assert_eq!(result_schema.field(0).name(), "text");
+    // Embedding columns follow in order
+    assert_eq!(result_schema.field(1).name(), "first");
+    assert_eq!(result_schema.field(2).name(), "second");
+    assert_eq!(result_schema.field(3).name(), "third");
+}
+
+#[test]
+fn test_embedding_error_propagation() {
+    // Test that errors from embedding computation are properly propagated
+    #[derive(Debug)]
+    struct FailingEmbed {
+        name: String,
+    }
+
+    impl EmbeddingFunction for FailingEmbed {
+        fn name(&self) -> &str {
+            &self.name
+        }
+
+        fn source_type(&self) -> Result<Cow<'_, DataType>> {
+            Ok(Cow::Owned(DataType::Utf8))
+        }
+
+        fn dest_type(&self) -> Result<Cow<'_, DataType>> {
+            Ok(Cow::Owned(DataType::new_fixed_size_list(
+                DataType::Float32,
+                2,
+                true,
+            )))
+        }
+
+        fn compute_source_embeddings(&self, _source: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+            Err(Error::Runtime {
+                message: "Intentional failure".to_string(),
+            })
+        }
+
+        fn compute_query_embeddings(&self, _input: Arc<dyn Array>) -> Result<Arc<dyn Array>> {
+            unimplemented!()
+        }
+    }
+
+    let batch = create_test_batch().unwrap();
+    let schema = batch.schema();
+
+    let embed = Arc::new(FailingEmbed {
+        name: "failing".to_string(),
+    });
+    let def = EmbeddingDefinition::new("text", "failing", Some("emb"));
+
+    let reader = RecordBatchIterator::new(vec![Ok(batch)], schema);
+    let embeddings = vec![(def, embed as Arc<dyn EmbeddingFunction>)];
+    let mut with_embeddings = WithEmbeddings::new(reader, embeddings);
+
+    let result = with_embeddings.next().unwrap();
+    assert!(result.is_err());
+    let err_msg = format!("{}", result.err().unwrap());
+    assert!(err_msg.contains("Intentional failure"));
+}
+
+#[test]
+fn test_maybe_embedded_with_no_embeddings() {
+    // Test that MaybeEmbedded::No variant works correctly
+    let batch = create_test_batch().unwrap();
+    let schema = batch.schema();
+
+    let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], schema.clone());
+    let table_def = lancedb::table::TableDefinition {
+        schema: schema.clone(),
+        column_definitions: vec![lancedb::table::ColumnDefinition {
+            kind: lancedb::table::ColumnKind::Physical,
+        }],
+    };
+
+    let mut maybe_embedded = MaybeEmbedded::try_new(reader, table_def, None).unwrap();
+
+    let result = maybe_embedded.next().unwrap().unwrap();
+    assert_eq!(result.num_columns(), 1);
+    assert_eq!(result.column(0).as_ref(), batch.column(0).as_ref());
+}