Compare commits

..

12 Commits

Author SHA1 Message Date
Jack Ye a3e3c1ad5c feat(rust): add namespace client properties 2026-04-15 22:58:05 -07:00
Jack Ye c6e43b545d refactor(rust): eagerly init listing namespace db 2026-04-15 22:28:52 -07:00
Jack Ye 5807ad464b refactor(rust): cache listing namespace db 2026-04-15 17:23:50 -07:00
Jack Ye 513c2215c5 feat(rust): support nested namespace ops in listing db 2026-04-15 17:15:26 -07:00
Gezi-lzq 10879d99b8 docs: fix broken documentation links (#3278) 2026-04-15 20:56:59 +08:00
Lance Release 4e6a1d5dce Bump version: 0.28.0-beta.4 → 0.28.0-beta.5 2026-04-12 23:51:14 +00:00
Lance Release 13d2759356 Bump version: 0.31.0-beta.4 → 0.31.0-beta.5 2026-04-12 23:50:50 +00:00
Jack Ye 7f52ec8c36 feat(python): support child namepsace operations and json serialization for LanceDBConnection (#3265)
## Summary

Add connection serialization and child namespace support to
`LanceDBConnection`.

- `DBConnection.serialize()` / `lancedb.deserialize()` for connection
reconstruction in remote workers
- Cache `namespace_client()` in `LanceDBConnection` to avoid repeated
DirectoryNamespace builds
- `LanceDBConnection` transparently delegates child namespace operations
(open_table, create_table, list_tables, drop_table, create_namespace,
etc.) to `LanceNamespaceDBConnection` via `_namespace_conn()`
- Root namespace operations still go through the original Rust path
- Generic worker property override mechanism: any
`namespace_client_properties` key prefixed with `_lancedb_worker_` has
the prefix stripped and overrides the corresponding property when
`deserialize(data, for_worker=True)`
- `LanceNamespaceDBConnection` stores
`namespace_client_impl`/`namespace_client_properties` for serialization
roundtrip

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-12 16:49:45 -07:00
Lance Release c6ae0de3ee Bump version: 0.28.0-beta.3 → 0.28.0-beta.4 2026-04-12 03:57:58 +00:00
Lance Release 231f0655ce Bump version: 0.31.0-beta.3 → 0.31.0-beta.4 2026-04-12 03:57:35 +00:00
LanceDB Robot 8c52977c59 chore: update lance dependency to v5.1.0-beta.3 (#3266)
## Summary
- Bump Rust Lance dependencies to `v5.1.0-beta.3` using
`ci/set_lance_version.py`.
- Update Java `lance-core.version` to `5.1.0-beta.3` in `java/pom.xml`.
- Refresh `Cargo.lock` metadata to the `v5.1.0-beta.3` Lance git tag.

## Verification
- `cargo clippy --workspace --tests --all-features -- -D warnings`
- `cargo fmt --all`

## Upstream Tag
- https://github.com/lance-format/lance/releases/tag/v5.1.0-beta.3
2026-04-11 20:56:49 -07:00
Lance Release 359710a0bf Bump version: 0.28.0-beta.2 → 0.28.0-beta.3 2026-04-11 22:44:52 +00:00
42 changed files with 782 additions and 281 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion] [tool.bumpversion]
current_version = "0.28.0-beta.2" current_version = "0.28.0-beta.5"
parse = """(?x) parse = """(?x)
(?P<major>0|[1-9]\\d*)\\. (?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\. (?P<minor>0|[1-9]\\d*)\\.
+1 -1
View File
@@ -18,6 +18,6 @@ body:
label: Link label: Link
description: > description: >
Provide a link to the existing documentation, if applicable. Provide a link to the existing documentation, if applicable.
placeholder: ex. https://lancedb.com/docs/tables/... placeholder: ex. https://docs.lancedb.com/tables/...
validations: validations:
required: false required: false
Generated
+35 -35
View File
@@ -3072,8 +3072,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]] [[package]]
name = "fsst" name = "fsst"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"rand 0.9.2", "rand 0.9.2",
@@ -4134,8 +4134,8 @@ dependencies = [
[[package]] [[package]]
name = "lance" name = "lance"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-arith", "arrow-arith",
@@ -4202,8 +4202,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-arrow" name = "lance-arrow"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4224,8 +4224,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-bitpacking" name = "lance-bitpacking"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrayref", "arrayref",
"paste", "paste",
@@ -4234,8 +4234,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-core" name = "lance-core"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4272,8 +4272,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-datafusion" name = "lance-datafusion"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-array", "arrow-array",
@@ -4304,8 +4304,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-datagen" name = "lance-datagen"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-array", "arrow-array",
@@ -4323,8 +4323,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-encoding" name = "lance-encoding"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-arith", "arrow-arith",
"arrow-array", "arrow-array",
@@ -4361,8 +4361,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-file" name = "lance-file"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-arith", "arrow-arith",
"arrow-array", "arrow-array",
@@ -4394,8 +4394,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-index" name = "lance-index"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-arith", "arrow-arith",
@@ -4459,8 +4459,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-io" name = "lance-io"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-arith", "arrow-arith",
@@ -4504,8 +4504,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-linalg" name = "lance-linalg"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4521,8 +4521,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-namespace" name = "lance-namespace"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"async-trait", "async-trait",
@@ -4535,8 +4535,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-namespace-impls" name = "lance-namespace-impls"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-ipc", "arrow-ipc",
@@ -4581,8 +4581,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-table" name = "lance-table"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-array", "arrow-array",
@@ -4621,8 +4621,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-testing" name = "lance-testing"
version = "5.1.0-beta.2" version = "5.1.0-beta.3"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0" source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-schema", "arrow-schema",
@@ -4633,7 +4633,7 @@ dependencies = [
[[package]] [[package]]
name = "lancedb" name = "lancedb"
version = "0.28.0-beta.2" version = "0.28.0-beta.5"
dependencies = [ dependencies = [
"ahash", "ahash",
"anyhow", "anyhow",
@@ -4715,7 +4715,7 @@ dependencies = [
[[package]] [[package]]
name = "lancedb-nodejs" name = "lancedb-nodejs"
version = "0.28.0-beta.2" version = "0.28.0-beta.5"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4737,7 +4737,7 @@ dependencies = [
[[package]] [[package]]
name = "lancedb-python" name = "lancedb-python"
version = "0.31.0-beta.2" version = "0.31.0-beta.5"
dependencies = [ dependencies = [
"arrow", "arrow",
"async-trait", "async-trait",
+14 -14
View File
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0" rust-version = "1.91.0"
[workspace.dependencies] [workspace.dependencies]
lance = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance = { "version" = "=5.1.0-beta.3", default-features = false, "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-core = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-datagen = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-file = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-io = { "version" = "=5.1.0-beta.3", default-features = false, "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-index = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-linalg = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-namespace = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-namespace-impls = { "version" = "=5.1.0-beta.3", default-features = false, "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-table = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-testing = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-datafusion = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-encoding = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" } lance-arrow = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8" ahash = "0.8"
# Note that this one does not include pyarrow # Note that this one does not include pyarrow
arrow = { version = "57.2", optional = false } arrow = { version = "57.2", optional = false }
+2 -2
View File
@@ -15,7 +15,7 @@
# **The Multimodal AI Lakehouse** # **The Multimodal AI Lakehouse**
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors) [**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
**The ultimate multimodal data platform for AI/ML applications.** **The ultimate multimodal data platform for AI/ML applications.**
@@ -57,7 +57,7 @@ LanceDB is a central location where developers can build, train and analyze thei
## **How to Install**: ## **How to Install**:
Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally. Follow the [Quickstart](https://docs.lancedb.com/quickstart) doc to set up LanceDB locally.
**API & SDK:** We also support Python, Typescript and Rust SDKs **API & SDK:** We also support Python, Typescript and Rust SDKs
+1 -1
View File
@@ -1,6 +1,6 @@
# LanceDB Documentation # LanceDB Documentation
LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs). LanceDB docs are available at [docs.lancedb.com](https://docs.lancedb.com).
The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml) The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
whenever a commit is pushed to the `main` branch. So it is possible for the docs to show whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency> <dependency>
<groupId>com.lancedb</groupId> <groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId> <artifactId>lancedb-core</artifactId>
<version>0.28.0-beta.2</version> <version>0.28.0-beta.5</version>
</dependency> </dependency>
``` ```
+1 -1
View File
@@ -34,7 +34,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
console.log(results); console.log(results);
``` ```
The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples. The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
## Development ## Development
+1 -1
View File
@@ -89,4 +89,4 @@ optional storageOptions: Record<string, string>;
(For LanceDB OSS only): configuration for object storage. (For LanceDB OSS only): configuration for object storage.
The available options are described at https://lancedb.com/docs/storage/ The available options are described at https://docs.lancedb.com/storage/
+1 -1
View File
@@ -97,4 +97,4 @@ Configuration for object storage.
Options already set on the connection will be inherited by the table, Options already set on the connection will be inherited by the table,
but can be overridden here. but can be overridden here.
The available options are described at https://lancedb.com/docs/storage/ The available options are described at https://docs.lancedb.com/storage/
+1 -1
View File
@@ -42,4 +42,4 @@ Configuration for object storage.
Options already set on the connection will be inherited by the table, Options already set on the connection will be inherited by the table,
but can be overridden here. but can be overridden here.
The available options are described at https://lancedb.com/docs/storage/ The available options are described at https://docs.lancedb.com/storage/
+1 -1
View File
@@ -8,7 +8,7 @@
<parent> <parent>
<groupId>com.lancedb</groupId> <groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId> <artifactId>lancedb-parent</artifactId>
<version>0.28.0-beta.2</version> <version>0.28.0-beta.5</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId> <groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId> <artifactId>lancedb-parent</artifactId>
<version>0.28.0-beta.2</version> <version>0.28.0-beta.5</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description> <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version> <arrow.version>15.0.0</arrow.version>
<lance-core.version>5.1.0-beta.2</lance-core.version> <lance-core.version>5.1.0-beta.3</lance-core.version>
<spotless.skip>false</spotless.skip> <spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version> <spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version> <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package] [package]
name = "lancedb-nodejs" name = "lancedb-nodejs"
edition.workspace = true edition.workspace = true
version = "0.28.0-beta.2" version = "0.28.0-beta.5"
license.workspace = true license.workspace = true
description.workspace = true description.workspace = true
repository.workspace = true repository.workspace = true
+1 -1
View File
@@ -30,7 +30,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
console.log(results); console.log(results);
``` ```
The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples. The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
## Development ## Development
+2 -2
View File
@@ -42,7 +42,7 @@ export interface CreateTableOptions {
* Options already set on the connection will be inherited by the table, * Options already set on the connection will be inherited by the table,
* but can be overridden here. * but can be overridden here.
* *
* The available options are described at https://lancedb.com/docs/storage/ * The available options are described at https://docs.lancedb.com/storage/
*/ */
storageOptions?: Record<string, string>; storageOptions?: Record<string, string>;
@@ -78,7 +78,7 @@ export interface OpenTableOptions {
* Options already set on the connection will be inherited by the table, * Options already set on the connection will be inherited by the table,
* but can be overridden here. * but can be overridden here.
* *
* The available options are described at https://lancedb.com/docs/storage/ * The available options are described at https://docs.lancedb.com/storage/
*/ */
storageOptions?: Record<string, string>; storageOptions?: Record<string, string>;
/** /**
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-darwin-arm64", "name": "@lancedb/lancedb-darwin-arm64",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": ["darwin"], "os": ["darwin"],
"cpu": ["arm64"], "cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node", "main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-arm64-gnu", "name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": ["linux"], "os": ["linux"],
"cpu": ["arm64"], "cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node", "main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-arm64-musl", "name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": ["linux"], "os": ["linux"],
"cpu": ["arm64"], "cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node", "main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-x64-gnu", "name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": ["linux"], "os": ["linux"],
"cpu": ["x64"], "cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node", "main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-x64-musl", "name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": ["linux"], "os": ["linux"],
"cpu": ["x64"], "cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node", "main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-win32-arm64-msvc", "name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": [ "os": [
"win32" "win32"
], ],
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-win32-x64-msvc", "name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"os": ["win32"], "os": ["win32"],
"cpu": ["x64"], "cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node", "main": "lancedb.win32-x64-msvc.node",
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "@lancedb/lancedb", "name": "@lancedb/lancedb",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@lancedb/lancedb", "name": "@lancedb/lancedb",
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"cpu": [ "cpu": [
"x64", "x64",
"arm64" "arm64"
+1 -1
View File
@@ -11,7 +11,7 @@
"ann" "ann"
], ],
"private": false, "private": false,
"version": "0.28.0-beta.2", "version": "0.28.0-beta.5",
"main": "dist/index.js", "main": "dist/index.js",
"exports": { "exports": {
".": "./dist/index.js", ".": "./dist/index.js",
+1 -1
View File
@@ -35,7 +35,7 @@ pub struct ConnectionOptions {
pub read_consistency_interval: Option<f64>, pub read_consistency_interval: Option<f64>,
/// (For LanceDB OSS only): configuration for object storage. /// (For LanceDB OSS only): configuration for object storage.
/// ///
/// The available options are described at https://lancedb.com/docs/storage/ /// The available options are described at https://docs.lancedb.com/storage/
pub storage_options: Option<HashMap<String, String>>, pub storage_options: Option<HashMap<String, String>>,
/// (For LanceDB OSS only): the session to use for this connection. Holds /// (For LanceDB OSS only): the session to use for this connection. Holds
/// shared caches and other session-specific state. /// shared caches and other session-specific state.
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion] [tool.bumpversion]
current_version = "0.31.0-beta.3" current_version = "0.31.0-beta.5"
parse = """(?x) parse = """(?x)
(?P<major>0|[1-9]\\d*)\\. (?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\. (?P<minor>0|[1-9]\\d*)\\.
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "lancedb-python" name = "lancedb-python"
version = "0.31.0-beta.3" version = "0.31.0-beta.5"
edition.workspace = true edition.workspace = true
description = "Python bindings for LanceDB" description = "Python bindings for LanceDB"
license.workspace = true license.workspace = true
+81 -2
View File
@@ -110,7 +110,7 @@ def connect(
default configuration is used. default configuration is used.
storage_options: dict, optional storage_options: dict, optional
Additional options for the storage backend. See available options at Additional options for the storage backend. See available options at
<https://lancedb.com/docs/storage/> <https://docs.lancedb.com/storage/>
session: Session, optional session: Session, optional
(For LanceDB OSS only) (For LanceDB OSS only)
A session to use for this connection. Sessions allow you to configure A session to use for this connection. Sessions allow you to configure
@@ -215,6 +215,85 @@ def connect(
) )
WORKER_PROPERTY_PREFIX = "_lancedb_worker_"
def _apply_worker_overrides(props: dict[str, str]) -> dict[str, str]:
"""Apply worker property overrides.
Any key starting with ``_lancedb_worker_`` is extracted, the prefix
is stripped, and the resulting key-value pair is put back into the
map (overriding the existing value if present). The original
prefixed key is removed.
"""
worker_keys = [k for k in props if k.startswith(WORKER_PROPERTY_PREFIX)]
if not worker_keys:
return props
result = dict(props)
for key in worker_keys:
value = result.pop(key)
real_key = key[len(WORKER_PROPERTY_PREFIX) :]
result[real_key] = value
return result
def deserialize_conn(
data: str,
*,
for_worker: bool = False,
) -> DBConnection:
"""Reconstruct a DBConnection from a serialized string.
The string must have been produced by
:meth:`DBConnection.serialize`.
Parameters
----------
data : str
String produced by ``serialize()``.
for_worker : bool, default False
When ``True``, any namespace client property whose key starts
with ``_lancedb_worker_`` has that prefix stripped and the
value overrides the corresponding property. For example,
``_lancedb_worker_uri`` replaces ``uri``.
Returns
-------
DBConnection
A new connection matching the serialized state.
"""
import json
parsed = json.loads(data)
connection_type = parsed.get("connection_type")
rci_secs = parsed.get("read_consistency_interval_seconds")
rci = timedelta(seconds=rci_secs) if rci_secs is not None else None
storage_options = parsed.get("storage_options")
if connection_type == "namespace":
props = dict(parsed.get("namespace_client_properties") or {})
if for_worker:
props = _apply_worker_overrides(props)
return connect_namespace(
namespace_client_impl=parsed["namespace_client_impl"],
namespace_client_properties=props,
read_consistency_interval=rci,
storage_options=storage_options,
namespace_client_pushdown_operations=parsed.get(
"namespace_client_pushdown_operations"
),
)
elif connection_type == "local":
return LanceDBConnection(
parsed["uri"],
read_consistency_interval=rci,
storage_options=storage_options,
)
else:
raise ValueError(f"Unknown connection_type: {connection_type}")
async def connect_async( async def connect_async(
uri: URI, uri: URI,
*, *,
@@ -257,7 +336,7 @@ async def connect_async(
default configuration is used. default configuration is used.
storage_options: dict, optional storage_options: dict, optional
Additional options for the storage backend. See available options at Additional options for the storage backend. See available options at
<https://lancedb.com/docs/storage/> <https://docs.lancedb.com/storage/>
session: Session, optional session: Session, optional
(For LanceDB OSS only) (For LanceDB OSS only)
A session to use for this connection. Sessions allow you to configure A session to use for this connection. Sessions allow you to configure
+1 -1
View File
@@ -96,7 +96,7 @@ def data_to_reader(
f"Unknown data type {type(data)}. " f"Unknown data type {type(data)}. "
"Supported types: list of dicts, pandas DataFrame, polars DataFrame, " "Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
"pyarrow Table/RecordBatch, or Pydantic models. " "pyarrow Table/RecordBatch, or Pydantic models. "
"See https://lancedb.com/docs/tables/ for examples." "See https://docs.lancedb.com/tables/ for examples."
) )
+103 -66
View File
@@ -282,7 +282,7 @@ class DBConnection(EnforceOverrides):
Additional options for the storage backend. Options already set on the Additional options for the storage backend. Options already set on the
connection will be inherited by the table, but can be overridden here. connection will be inherited by the table, but can be overridden here.
See available options at See available options at
<https://lancedb.com/docs/storage/> <https://docs.lancedb.com/storage/>
To enable stable row IDs (row IDs remain stable after compaction, To enable stable row IDs (row IDs remain stable after compaction,
update, delete, and merges), set `new_table_enable_stable_row_ids` update, delete, and merges), set `new_table_enable_stable_row_ids`
@@ -433,7 +433,7 @@ class DBConnection(EnforceOverrides):
Additional options for the storage backend. Options already set on the Additional options for the storage backend. Options already set on the
connection will be inherited by the table, but can be overridden here. connection will be inherited by the table, but can be overridden here.
See available options at See available options at
<https://lancedb.com/docs/storage/> <https://docs.lancedb.com/storage/>
Returns Returns
------- -------
@@ -529,6 +529,19 @@ class DBConnection(EnforceOverrides):
"namespace_client is not supported for this connection type" "namespace_client is not supported for this connection type"
) )
def serialize(self) -> str:
"""Serialize this connection for reconstruction.
The returned string can be passed to :func:`lancedb.deserialize_conn`
to recreate an equivalent connection, e.g. in a remote worker.
Returns
-------
str
Serialized representation of this connection.
"""
raise NotImplementedError("serialize is not supported for this connection type")
class LanceDBConnection(DBConnection): class LanceDBConnection(DBConnection):
""" """
@@ -581,6 +594,7 @@ class LanceDBConnection(DBConnection):
): ):
if _inner is not None: if _inner is not None:
self._conn = _inner self._conn = _inner
self._cached_namespace_client = None
return return
if not isinstance(uri, Path): if not isinstance(uri, Path):
@@ -628,6 +642,7 @@ class LanceDBConnection(DBConnection):
# beyond _conn. # beyond _conn.
self.storage_options = storage_options self.storage_options = storage_options
self._conn = AsyncConnection(LOOP.run(do_connect())) self._conn = AsyncConnection(LOOP.run(do_connect()))
self._cached_namespace_client: Optional[LanceNamespace] = None
@property @property
def read_consistency_interval(self) -> Optional[timedelta]: def read_consistency_interval(self) -> Optional[timedelta]:
@@ -652,6 +667,22 @@ class LanceDBConnection(DBConnection):
val += ")" val += ")"
return val return val
@override
def serialize(self) -> str:
import json
rci = self.read_consistency_interval
return json.dumps(
{
"connection_type": "local",
"uri": self.uri,
"storage_options": self.storage_options,
"read_consistency_interval_seconds": (
rci.total_seconds() if rci else None
),
}
)
async def _async_get_table_names(self, start_after: Optional[str], limit: int): async def _async_get_table_names(self, start_after: Optional[str], limit: int):
conn = AsyncConnection(await lancedb_connect(self.uri)) conn = AsyncConnection(await lancedb_connect(self.uri))
return await conn.table_names(start_after=start_after, limit=limit) return await conn.table_names(start_after=start_after, limit=limit)
@@ -687,10 +718,10 @@ class LanceDBConnection(DBConnection):
""" """
if namespace_path is None: if namespace_path is None:
namespace_path = [] namespace_path = []
return LOOP.run( return self._namespace_conn().list_namespaces(
self._conn.list_namespaces( namespace_path=namespace_path,
namespace_path=namespace_path, page_token=page_token, limit=limit page_token=page_token,
) limit=limit,
) )
@override @override
@@ -700,27 +731,10 @@ class LanceDBConnection(DBConnection):
mode: Optional[str] = None, mode: Optional[str] = None,
properties: Optional[Dict[str, str]] = None, properties: Optional[Dict[str, str]] = None,
) -> CreateNamespaceResponse: ) -> CreateNamespaceResponse:
"""Create a new namespace. return self._namespace_conn().create_namespace(
namespace_path=namespace_path,
Parameters mode=mode,
---------- properties=properties,
namespace_path: List[str]
The namespace identifier to create.
mode: str, optional
Creation mode - "create" (fail if exists), "exist_ok" (skip if exists),
or "overwrite" (replace if exists). Case insensitive.
properties: Dict[str, str], optional
Properties to set on the namespace.
Returns
-------
CreateNamespaceResponse
Response containing the properties of the created namespace.
"""
return LOOP.run(
self._conn.create_namespace(
namespace_path=namespace_path, mode=mode, properties=properties
)
) )
@override @override
@@ -730,46 +744,19 @@ class LanceDBConnection(DBConnection):
mode: Optional[str] = None, mode: Optional[str] = None,
behavior: Optional[str] = None, behavior: Optional[str] = None,
) -> DropNamespaceResponse: ) -> DropNamespaceResponse:
"""Drop a namespace. return self._namespace_conn().drop_namespace(
namespace_path=namespace_path,
Parameters mode=mode,
---------- behavior=behavior,
namespace_path: List[str]
The namespace identifier to drop.
mode: str, optional
Whether to skip if not exists ("SKIP") or fail ("FAIL"). Case insensitive.
behavior: str, optional
Whether to restrict drop if not empty ("RESTRICT") or cascade ("CASCADE").
Case insensitive.
Returns
-------
DropNamespaceResponse
Response containing properties and transaction_id if applicable.
"""
return LOOP.run(
self._conn.drop_namespace(
namespace_path=namespace_path, mode=mode, behavior=behavior
)
) )
@override @override
def describe_namespace( def describe_namespace(
self, namespace_path: List[str] self, namespace_path: List[str]
) -> DescribeNamespaceResponse: ) -> DescribeNamespaceResponse:
"""Describe a namespace. return self._namespace_conn().describe_namespace(
namespace_path=namespace_path,
Parameters )
----------
namespace_path: List[str]
The namespace identifier to describe.
Returns
-------
DescribeNamespaceResponse
Response containing the namespace properties.
"""
return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path))
@override @override
def list_tables( def list_tables(
@@ -798,6 +785,12 @@ class LanceDBConnection(DBConnection):
""" """
if namespace_path is None: if namespace_path is None:
namespace_path = [] namespace_path = []
if namespace_path:
return self._namespace_conn().list_tables(
namespace_path=namespace_path,
page_token=page_token,
limit=limit,
)
return LOOP.run( return LOOP.run(
self._conn.list_tables( self._conn.list_tables(
namespace_path=namespace_path, page_token=page_token, limit=limit namespace_path=namespace_path, page_token=page_token, limit=limit
@@ -886,6 +879,22 @@ class LanceDBConnection(DBConnection):
raise ValueError("mode must be either 'create' or 'overwrite'") raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name) validate_table_name(name)
if namespace_path:
return self._namespace_conn().create_table(
name,
data=data,
schema=schema,
mode=mode,
exist_ok=exist_ok,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
namespace_path=namespace_path,
storage_options=storage_options,
data_storage_version=data_storage_version,
enable_v2_manifest_paths=enable_v2_manifest_paths,
)
tbl = LanceTable.create( tbl = LanceTable.create(
self, self,
name, name,
@@ -901,6 +910,19 @@ class LanceDBConnection(DBConnection):
) )
return tbl return tbl
def _namespace_conn(self) -> DBConnection:
"""Return a LanceNamespaceDBConnection backed by this connection's
directory namespace. Used to delegate child-namespace operations."""
from lancedb.namespace import LanceNamespaceDBConnection
return LanceNamespaceDBConnection(
self.namespace_client(),
read_consistency_interval=self.read_consistency_interval,
storage_options=self.storage_options,
namespace_client_impl=None,
namespace_client_properties=None,
)
@override @override
def open_table( def open_table(
self, self,
@@ -917,7 +939,8 @@ class LanceDBConnection(DBConnection):
name: str name: str
The name of the table. The name of the table.
namespace_path: List[str], optional namespace_path: List[str], optional
The namespace to open the table from. The namespace to open the table from. When non-empty, the
table is resolved through the directory namespace client.
Returns Returns
------- -------
@@ -936,6 +959,14 @@ class LanceDBConnection(DBConnection):
stacklevel=2, stacklevel=2,
) )
if namespace_path:
return self._namespace_conn().open_table(
name,
namespace_path=namespace_path,
storage_options=storage_options,
index_cache_size=index_cache_size,
)
return LanceTable.open( return LanceTable.open(
self, self,
name, name,
@@ -1020,6 +1051,9 @@ class LanceDBConnection(DBConnection):
""" """
if namespace_path is None: if namespace_path is None:
namespace_path = [] namespace_path = []
if namespace_path:
self._namespace_conn().drop_table(name, namespace_path=namespace_path)
return
LOOP.run( LOOP.run(
self._conn.drop_table( self._conn.drop_table(
name, namespace_path=namespace_path, ignore_missing=ignore_missing name, namespace_path=namespace_path, ignore_missing=ignore_missing
@@ -1071,14 +1105,17 @@ class LanceDBConnection(DBConnection):
"""Get the equivalent namespace client for this connection. """Get the equivalent namespace client for this connection.
Returns a DirectoryNamespace pointing to the same root with the Returns a DirectoryNamespace pointing to the same root with the
same storage options. same storage options. The result is cached for the lifetime of
this connection.
Returns Returns
------- -------
LanceNamespace LanceNamespace
The namespace client for this connection. The namespace client for this connection.
""" """
return LOOP.run(self._conn.namespace_client()) if self._cached_namespace_client is None:
self._cached_namespace_client = LOOP.run(self._conn.namespace_client())
return self._cached_namespace_client
@deprecation.deprecated( @deprecation.deprecated(
deprecated_in="0.15.1", deprecated_in="0.15.1",
@@ -1397,7 +1434,7 @@ class AsyncConnection(object):
Additional options for the storage backend. Options already set on the Additional options for the storage backend. Options already set on the
connection will be inherited by the table, but can be overridden here. connection will be inherited by the table, but can be overridden here.
See available options at See available options at
<https://lancedb.com/docs/storage/> <https://docs.lancedb.com/storage/>
To enable stable row IDs (row IDs remain stable after compaction, To enable stable row IDs (row IDs remain stable after compaction,
update, delete, and merges), set `new_table_enable_stable_row_ids` update, delete, and merges), set `new_table_enable_stable_row_ids`
@@ -1588,7 +1625,7 @@ class AsyncConnection(object):
Additional options for the storage backend. Options already set on the Additional options for the storage backend. Options already set on the
connection will be inherited by the table, but can be overridden here. connection will be inherited by the table, but can be overridden here.
See available options at See available options at
<https://lancedb.com/docs/storage/> <https://docs.lancedb.com/storage/>
index_cache_size: int, default 256 index_cache_size: int, default 256
**Deprecated**: Use session-level cache configuration instead. **Deprecated**: Use session-level cache configuration instead.
Create a Session with custom cache sizes and pass it to lancedb.connect(). Create a Session with custom cache sizes and pass it to lancedb.connect().
+53 -10
View File
@@ -381,6 +381,8 @@ class LanceNamespaceDBConnection(DBConnection):
storage_options: Optional[Dict[str, str]] = None, storage_options: Optional[Dict[str, str]] = None,
session: Optional[Session] = None, session: Optional[Session] = None,
namespace_client_pushdown_operations: Optional[List[str]] = None, namespace_client_pushdown_operations: Optional[List[str]] = None,
namespace_client_impl: Optional[str] = None,
namespace_client_properties: Optional[Dict[str, str]] = None,
): ):
""" """
Initialize a namespace-based LanceDB connection. Initialize a namespace-based LanceDB connection.
@@ -406,12 +408,43 @@ class LanceNamespaceDBConnection(DBConnection):
namespace.create_table() instead of using declare_table + local write. namespace.create_table() instead of using declare_table + local write.
Default is None (no pushdown, all operations run locally). Default is None (no pushdown, all operations run locally).
namespace_client_impl : Optional[str]
The namespace implementation name used to create this connection.
Stored for serialization purposes.
namespace_client_properties : Optional[Dict[str, str]]
The namespace properties used to create this connection.
Stored for serialization purposes.
""" """
self._namespace_client = namespace_client self._namespace_client = namespace_client
self.read_consistency_interval = read_consistency_interval self.read_consistency_interval = read_consistency_interval
self.storage_options = storage_options or {} self.storage_options = storage_options or {}
self.session = session self.session = session
self._pushdown_operations = set(namespace_client_pushdown_operations or []) self._namespace_client_pushdown_operations = set(
namespace_client_pushdown_operations or []
)
self._namespace_client_impl = namespace_client_impl
self._namespace_client_properties = namespace_client_properties
@override
def serialize(self) -> str:
import json
return json.dumps(
{
"connection_type": "namespace",
"namespace_client_impl": self._namespace_client_impl,
"namespace_client_properties": self._namespace_client_properties,
"namespace_client_pushdown_operations": sorted(
self._namespace_client_pushdown_operations
),
"storage_options": self.storage_options or None,
"read_consistency_interval_seconds": (
self.read_consistency_interval.total_seconds()
if self.read_consistency_interval
else None
),
}
)
@override @override
def table_names( def table_names(
@@ -467,7 +500,7 @@ class LanceNamespaceDBConnection(DBConnection):
table_id = namespace_path + [name] table_id = namespace_path + [name]
if "CreateTable" in self._pushdown_operations: if "CreateTable" in self._namespace_client_pushdown_operations:
return self._create_table_server_side( return self._create_table_server_side(
name=name, name=name,
data=data, data=data,
@@ -549,7 +582,7 @@ class LanceNamespaceDBConnection(DBConnection):
storage_options=merged_storage_options, storage_options=merged_storage_options,
location=location, location=location,
namespace_client=self._namespace_client, namespace_client=self._namespace_client,
pushdown_operations=self._pushdown_operations, pushdown_operations=self._namespace_client_pushdown_operations,
) )
return tbl return tbl
@@ -580,10 +613,13 @@ class LanceNamespaceDBConnection(DBConnection):
fill_value=fill_value, fill_value=fill_value,
) )
merged = dict(self.storage_options or {})
if storage_options:
merged.update(storage_options)
request = CreateTableRequest( request = CreateTableRequest(
id=table_id, id=table_id,
mode=_normalize_create_table_mode(mode), mode=_normalize_create_table_mode(mode),
properties=self.storage_options if self.storage_options else None, properties=merged or None,
) )
try: try:
@@ -887,7 +923,7 @@ class LanceNamespaceDBConnection(DBConnection):
location=table_uri, location=table_uri,
namespace_client=namespace_client, namespace_client=namespace_client,
managed_versioning=managed_versioning, managed_versioning=managed_versioning,
pushdown_operations=self._pushdown_operations, pushdown_operations=self._namespace_client_pushdown_operations,
) )
@override @override
@@ -951,7 +987,9 @@ class AsyncLanceNamespaceDBConnection:
self.read_consistency_interval = read_consistency_interval self.read_consistency_interval = read_consistency_interval
self.storage_options = storage_options or {} self.storage_options = storage_options or {}
self.session = session self.session = session
self._pushdown_operations = set(namespace_client_pushdown_operations or []) self._namespace_client_pushdown_operations = set(
namespace_client_pushdown_operations or []
)
async def table_names( async def table_names(
self, self,
@@ -1006,7 +1044,7 @@ class AsyncLanceNamespaceDBConnection:
table_id = namespace_path + [name] table_id = namespace_path + [name]
if "CreateTable" in self._pushdown_operations: if "CreateTable" in self._namespace_client_pushdown_operations:
return await self._create_table_server_side( return await self._create_table_server_side(
name=name, name=name,
data=data, data=data,
@@ -1086,7 +1124,7 @@ class AsyncLanceNamespaceDBConnection:
storage_options=merged_storage_options, storage_options=merged_storage_options,
location=location, location=location,
namespace_client=self._namespace_client, namespace_client=self._namespace_client,
pushdown_operations=self._pushdown_operations, pushdown_operations=self._namespace_client_pushdown_operations,
) )
lance_table = await asyncio.to_thread(_create_table) lance_table = await asyncio.to_thread(_create_table)
@@ -1120,10 +1158,13 @@ class AsyncLanceNamespaceDBConnection:
fill_value=fill_value, fill_value=fill_value,
) )
merged = dict(self.storage_options or {})
if storage_options:
merged.update(storage_options)
request = CreateTableRequest( request = CreateTableRequest(
id=table_id, id=table_id,
mode=_normalize_create_table_mode(mode), mode=_normalize_create_table_mode(mode),
properties=self.storage_options if self.storage_options else None, properties=merged or None,
) )
self._namespace_client.create_table(request, arrow_ipc_bytes) self._namespace_client.create_table(request, arrow_ipc_bytes)
@@ -1190,7 +1231,7 @@ class AsyncLanceNamespaceDBConnection:
location=response.location, location=response.location,
namespace_client=self._namespace_client, namespace_client=self._namespace_client,
managed_versioning=managed_versioning, managed_versioning=managed_versioning,
pushdown_operations=self._pushdown_operations, pushdown_operations=self._namespace_client_pushdown_operations,
) )
lance_table = await asyncio.to_thread(_open_table) lance_table = await asyncio.to_thread(_open_table)
@@ -1472,6 +1513,8 @@ def connect_namespace(
storage_options=storage_options, storage_options=storage_options,
session=session, session=session,
namespace_client_pushdown_operations=namespace_client_pushdown_operations, namespace_client_pushdown_operations=namespace_client_pushdown_operations,
namespace_client_impl=namespace_client_impl,
namespace_client_properties=namespace_client_properties,
) )
+1 -1
View File
@@ -191,7 +191,7 @@ def _into_pyarrow_reader(
f"Unknown data type {type(data)}. " f"Unknown data type {type(data)}. "
"Supported types: list of dicts, pandas DataFrame, polars DataFrame, " "Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
"pyarrow Table/RecordBatch, or Pydantic models. " "pyarrow Table/RecordBatch, or Pydantic models. "
"See https://lancedb.com/docs/tables/ for examples." "See https://docs.lancedb.com/tables/ for examples."
) )
+11 -31
View File
@@ -897,42 +897,22 @@ def test_bypass_vector_index_sync(tmp_db: lancedb.DBConnection):
def test_local_namespace_operations(tmp_path): def test_local_namespace_operations(tmp_path):
"""Test that local mode namespace operations behave as expected.""" """Test that local mode namespace operations work via directory namespace."""
# Create a local database connection
db = lancedb.connect(tmp_path) db = lancedb.connect(tmp_path)
# Test list_namespaces returns empty list for root namespace # Root namespace starts empty
namespaces = db.list_namespaces().namespaces assert db.list_namespaces().namespaces == []
assert namespaces == []
# Test list_namespaces with non-empty namespace raises NotImplementedError # Create and list child namespace
with pytest.raises( db.create_namespace(["child"])
NotImplementedError, assert "child" in db.list_namespaces().namespaces
match="Namespace operations are not supported for listing database",
):
db.list_namespaces(namespace_path=["test"])
# List namespaces under child
assert db.list_namespaces(namespace_path=["child"]).namespaces == []
def test_local_create_namespace_not_supported(tmp_path): # Drop namespace
"""Test that create_namespace is not supported in local mode.""" db.drop_namespace(["child"])
db = lancedb.connect(tmp_path) assert db.list_namespaces().namespaces == []
with pytest.raises(
NotImplementedError,
match="Namespace operations are not supported for listing database",
):
db.create_namespace(["test_namespace"])
def test_local_drop_namespace_not_supported(tmp_path):
"""Test that drop_namespace is not supported in local mode."""
db = lancedb.connect(tmp_path)
with pytest.raises(
NotImplementedError,
match="Namespace operations are not supported for listing database",
):
db.drop_namespace(["test_namespace"])
def test_clone_table_latest_version(tmp_path): def test_clone_table_latest_version(tmp_path):
+8 -8
View File
@@ -681,7 +681,7 @@ class TestPushdownOperations:
{"root": self.temp_dir}, {"root": self.temp_dir},
namespace_client_pushdown_operations=["QueryTable"], namespace_client_pushdown_operations=["QueryTable"],
) )
assert "QueryTable" in db._pushdown_operations assert "QueryTable" in db._namespace_client_pushdown_operations
def test_create_table_pushdown_stored(self): def test_create_table_pushdown_stored(self):
"""Test that CreateTable pushdown is stored on sync connection.""" """Test that CreateTable pushdown is stored on sync connection."""
@@ -690,7 +690,7 @@ class TestPushdownOperations:
{"root": self.temp_dir}, {"root": self.temp_dir},
namespace_client_pushdown_operations=["CreateTable"], namespace_client_pushdown_operations=["CreateTable"],
) )
assert "CreateTable" in db._pushdown_operations assert "CreateTable" in db._namespace_client_pushdown_operations
def test_both_pushdowns_stored(self): def test_both_pushdowns_stored(self):
"""Test that both pushdown operations can be set together.""" """Test that both pushdown operations can be set together."""
@@ -699,13 +699,13 @@ class TestPushdownOperations:
{"root": self.temp_dir}, {"root": self.temp_dir},
namespace_client_pushdown_operations=["QueryTable", "CreateTable"], namespace_client_pushdown_operations=["QueryTable", "CreateTable"],
) )
assert "QueryTable" in db._pushdown_operations assert "QueryTable" in db._namespace_client_pushdown_operations
assert "CreateTable" in db._pushdown_operations assert "CreateTable" in db._namespace_client_pushdown_operations
def test_pushdown_defaults_to_empty(self): def test_pushdown_defaults_to_empty(self):
"""Test that pushdown operations default to empty.""" """Test that pushdown operations default to empty."""
db = lancedb.connect_namespace("dir", {"root": self.temp_dir}) db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
assert len(db._pushdown_operations) == 0 assert len(db._namespace_client_pushdown_operations) == 0
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -727,7 +727,7 @@ class TestAsyncPushdownOperations:
{"root": self.temp_dir}, {"root": self.temp_dir},
namespace_client_pushdown_operations=["QueryTable"], namespace_client_pushdown_operations=["QueryTable"],
) )
assert "QueryTable" in db._pushdown_operations assert "QueryTable" in db._namespace_client_pushdown_operations
async def test_async_create_table_pushdown_stored(self): async def test_async_create_table_pushdown_stored(self):
"""Test that CreateTable pushdown is stored on async connection.""" """Test that CreateTable pushdown is stored on async connection."""
@@ -736,9 +736,9 @@ class TestAsyncPushdownOperations:
{"root": self.temp_dir}, {"root": self.temp_dir},
namespace_client_pushdown_operations=["CreateTable"], namespace_client_pushdown_operations=["CreateTable"],
) )
assert "CreateTable" in db._pushdown_operations assert "CreateTable" in db._namespace_client_pushdown_operations
async def test_async_pushdown_defaults_to_empty(self): async def test_async_pushdown_defaults_to_empty(self):
"""Test that pushdown operations default to empty on async connection.""" """Test that pushdown operations default to empty on async connection."""
db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir}) db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
assert len(db._pushdown_operations) == 0 assert len(db._namespace_client_pushdown_operations) == 0
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "lancedb" name = "lancedb"
version = "0.28.0-beta.2" version = "0.28.0-beta.5"
edition.workspace = true edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications" description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true license.workspace = true
+94 -7
View File
@@ -171,7 +171,7 @@ impl OpenTableBuilder {
/// Options already set on the connection will be inherited by the table, /// Options already set on the connection will be inherited by the table,
/// but can be overridden here. /// but can be overridden here.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self { pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
let store_params = self let store_params = self
.request .request
@@ -188,7 +188,7 @@ impl OpenTableBuilder {
/// Options already set on the connection will be inherited by the table, /// Options already set on the connection will be inherited by the table,
/// but can be overridden here. /// but can be overridden here.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_options( pub fn storage_options(
mut self, mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>, pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
@@ -582,6 +582,14 @@ pub struct ConnectRequest {
/// Database specific options /// Database specific options
pub options: HashMap<String, String>, pub options: HashMap<String, String>,
/// Extra properties for the equivalent namespace client.
///
/// For a local [`ListingDatabase`], these are merged into the backing
/// `DirectoryNamespace` properties. This is useful for namespace-specific
/// settings such as `table_version_tracking_enabled` that are distinct from
/// storage options.
pub namespace_client_properties: HashMap<String, String>,
/// The interval at which to check for updates from other processes. /// The interval at which to check for updates from other processes.
/// ///
/// If None, then consistency is not checked. For performance /// If None, then consistency is not checked. For performance
@@ -621,6 +629,7 @@ impl ConnectBuilder {
client_config: Default::default(), client_config: Default::default(),
read_consistency_interval: None, read_consistency_interval: None,
options: HashMap::new(), options: HashMap::new(),
namespace_client_properties: HashMap::new(),
session: None, session: None,
}, },
embedding_registry: None, embedding_registry: None,
@@ -738,7 +747,7 @@ impl ConnectBuilder {
/// Set an option for the storage layer. /// Set an option for the storage layer.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self { pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.request.options.insert(key.into(), value.into()); self.request.options.insert(key.into(), value.into());
self self
@@ -746,7 +755,7 @@ impl ConnectBuilder {
/// Set multiple options for the storage layer. /// Set multiple options for the storage layer.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_options( pub fn storage_options(
mut self, mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>, pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
@@ -757,6 +766,31 @@ impl ConnectBuilder {
self self
} }
/// Set an additional property for the equivalent namespace client.
pub fn namespace_client_property(
mut self,
key: impl Into<String>,
value: impl Into<String>,
) -> Self {
self.request
.namespace_client_properties
.insert(key.into(), value.into());
self
}
/// Set multiple additional properties for the equivalent namespace client.
pub fn namespace_client_properties(
mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
) -> Self {
for (key, value) in pairs {
self.request
.namespace_client_properties
.insert(key.into(), value.into());
}
self
}
/// The interval at which to check for updates from other processes. This /// The interval at which to check for updates from other processes. This
/// only affects LanceDB OSS. /// only affects LanceDB OSS.
/// ///
@@ -893,6 +927,7 @@ pub struct ConnectNamespaceBuilder {
ns_impl: String, ns_impl: String,
properties: HashMap<String, String>, properties: HashMap<String, String>,
storage_options: HashMap<String, String>, storage_options: HashMap<String, String>,
namespace_client_properties: HashMap<String, String>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
embedding_registry: Option<Arc<dyn EmbeddingRegistry>>, embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
session: Option<Arc<lance::session::Session>>, session: Option<Arc<lance::session::Session>>,
@@ -905,6 +940,7 @@ impl ConnectNamespaceBuilder {
ns_impl: ns_impl.to_string(), ns_impl: ns_impl.to_string(),
properties, properties,
storage_options: HashMap::new(), storage_options: HashMap::new(),
namespace_client_properties: HashMap::new(),
read_consistency_interval: None, read_consistency_interval: None,
embedding_registry: None, embedding_registry: None,
session: None, session: None,
@@ -914,7 +950,7 @@ impl ConnectNamespaceBuilder {
/// Set an option for the storage layer. /// Set an option for the storage layer.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self { pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.storage_options.insert(key.into(), value.into()); self.storage_options.insert(key.into(), value.into());
self self
@@ -922,7 +958,7 @@ impl ConnectNamespaceBuilder {
/// Set multiple options for the storage layer. /// Set multiple options for the storage layer.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_options( pub fn storage_options(
mut self, mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>, pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
@@ -933,6 +969,29 @@ impl ConnectNamespaceBuilder {
self self
} }
/// Set an additional namespace client property.
pub fn namespace_client_property(
mut self,
key: impl Into<String>,
value: impl Into<String>,
) -> Self {
self.namespace_client_properties
.insert(key.into(), value.into());
self
}
/// Set multiple additional namespace client properties.
pub fn namespace_client_properties(
mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
) -> Self {
for (key, value) in pairs {
self.namespace_client_properties
.insert(key.into(), value.into());
}
self
}
/// The interval at which to check for updates from other processes. /// The interval at which to check for updates from other processes.
/// ///
/// If left unset, consistency is not checked. For maximum read /// If left unset, consistency is not checked. For maximum read
@@ -994,10 +1053,13 @@ impl ConnectNamespaceBuilder {
pub async fn execute(self) -> Result<Connection> { pub async fn execute(self) -> Result<Connection> {
use crate::database::namespace::LanceNamespaceDatabase; use crate::database::namespace::LanceNamespaceDatabase;
let mut properties = self.properties;
properties.extend(self.namespace_client_properties);
let internal = Arc::new( let internal = Arc::new(
LanceNamespaceDatabase::connect( LanceNamespaceDatabase::connect(
&self.ns_impl, &self.ns_impl,
self.properties, properties,
self.storage_options, self.storage_options,
self.read_consistency_interval, self.read_consistency_interval,
self.session, self.session,
@@ -1117,6 +1179,31 @@ mod tests {
assert_eq!(db.uri(), relative_uri.to_str().unwrap().to_string()); assert_eq!(db.uri(), relative_uri.to_str().unwrap().to_string());
} }
#[tokio::test]
async fn test_connect_with_namespace_client_properties() {
let tmp_dir = tempdir().unwrap();
let uri = tmp_dir.path().to_str().unwrap();
let db = connect(uri)
.namespace_client_property("table_version_tracking_enabled", "true")
.namespace_client_property("manifest_enabled", "true")
.execute()
.await
.unwrap();
let (ns_impl, properties) = db.namespace_client_config().await.unwrap();
assert_eq!(ns_impl, "dir");
assert_eq!(properties.get("root"), Some(&uri.to_string()));
assert_eq!(
properties.get("table_version_tracking_enabled"),
Some(&"true".to_string())
);
assert_eq!(
properties.get("manifest_enabled"),
Some(&"true".to_string())
);
}
#[tokio::test] #[tokio::test]
async fn test_table_names() { async fn test_table_names() {
let tc = new_test_connection().await.unwrap(); let tc = new_test_connection().await.unwrap();
+2 -2
View File
@@ -55,7 +55,7 @@ impl CreateTableBuilder {
/// Options already set on the connection will be inherited by the table, /// Options already set on the connection will be inherited by the table,
/// but can be overridden here. /// but can be overridden here.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self { pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
let store_params = self let store_params = self
.request .request
@@ -73,7 +73,7 @@ impl CreateTableBuilder {
/// Options already set on the connection will be inherited by the table, /// Options already set on the connection will be inherited by the table,
/// but can be overridden here. /// but can be overridden here.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_options( pub fn storage_options(
mut self, mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>, pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
+305 -71
View File
@@ -20,6 +20,7 @@ use snafu::ResultExt;
use crate::connection::ConnectRequest; use crate::connection::ConnectRequest;
use crate::database::ReadConsistency; use crate::database::ReadConsistency;
use crate::database::namespace::LanceNamespaceDatabase;
use crate::error::{CreateDirSnafu, Error, Result}; use crate::error::{CreateDirSnafu, Error, Result};
use crate::io::object_store::MirroringObjectStoreWrapper; use crate::io::object_store::MirroringObjectStoreWrapper;
use crate::table::NativeTable; use crate::table::NativeTable;
@@ -73,7 +74,7 @@ pub struct ListingDatabaseOptions {
/// These are used to create/list tables and they are inherited by all tables /// These are used to create/list tables and they are inherited by all tables
/// opened by this database. /// opened by this database.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub storage_options: HashMap<String, String>, pub storage_options: HashMap<String, String>,
} }
@@ -185,7 +186,7 @@ impl ListingDatabaseOptionsBuilder {
/// Set an option for the storage layer. /// Set an option for the storage layer.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self { pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.options self.options
.storage_options .storage_options
@@ -195,7 +196,7 @@ impl ListingDatabaseOptionsBuilder {
/// Set multiple options for the storage layer. /// Set multiple options for the storage layer.
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
pub fn storage_options( pub fn storage_options(
mut self, mut self,
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>, pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
@@ -255,6 +256,9 @@ pub struct ListingDatabase {
// Session for object stores and caching // Session for object stores and caching
session: Arc<lance::session::Session>, session: Arc<lance::session::Session>,
// Namespace-backed database for child namespace operations
namespace_database: Arc<LanceNamespaceDatabase>,
} }
impl std::fmt::Display for ListingDatabase { impl std::fmt::Display for ListingDatabase {
@@ -281,6 +285,44 @@ const MIRRORED_STORE: &str = "mirroredStore";
/// A connection to LanceDB /// A connection to LanceDB
impl ListingDatabase { impl ListingDatabase {
fn build_namespace_client_properties(
uri: &str,
storage_options: &HashMap<String, String>,
namespace_client_properties: HashMap<String, String>,
) -> HashMap<String, String> {
let mut properties = namespace_client_properties;
properties.insert("root".to_string(), uri.to_string());
for (key, value) in storage_options {
properties.insert(format!("storage.{}", key), value.clone());
}
properties
}
async fn connect_namespace_database(
uri: &str,
storage_options: HashMap<String, String>,
namespace_client_properties: HashMap<String, String>,
read_consistency_interval: Option<std::time::Duration>,
session: Arc<lance::session::Session>,
) -> Result<Arc<LanceNamespaceDatabase>> {
let ns_properties = Self::build_namespace_client_properties(
uri,
&storage_options,
namespace_client_properties,
);
Ok(Arc::new(
LanceNamespaceDatabase::connect(
"dir",
ns_properties,
storage_options,
read_consistency_interval,
Some(session),
HashSet::new(),
)
.await?,
))
}
/// Connect to a listing database /// Connect to a listing database
/// ///
/// The URI should be a path to a directory where the tables are stored. /// The URI should be a path to a directory where the tables are stored.
@@ -300,6 +342,7 @@ impl ListingDatabase {
uri, uri,
request.read_consistency_interval, request.read_consistency_interval,
options.new_table_config, options.new_table_config,
request.namespace_client_properties.clone(),
request.session.clone(), request.session.clone(),
) )
.await .await
@@ -387,6 +430,15 @@ impl ListingDatabase {
None => None, None => None,
}; };
let namespace_database = Self::connect_namespace_database(
&table_base_uri,
options.storage_options.clone(),
request.namespace_client_properties.clone(),
request.read_consistency_interval,
session.clone(),
)
.await?;
Ok(Self { Ok(Self {
uri: table_base_uri, uri: table_base_uri,
query_string, query_string,
@@ -398,6 +450,7 @@ impl ListingDatabase {
storage_options_provider: None, storage_options_provider: None,
new_table_config: options.new_table_config, new_table_config: options.new_table_config,
session, session,
namespace_database,
}) })
} }
Err(_) => { Err(_) => {
@@ -405,6 +458,7 @@ impl ListingDatabase {
uri, uri,
request.read_consistency_interval, request.read_consistency_interval,
options.new_table_config, options.new_table_config,
request.namespace_client_properties.clone(),
request.session.clone(), request.session.clone(),
) )
.await .await
@@ -416,6 +470,7 @@ impl ListingDatabase {
path: &str, path: &str,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
new_table_config: NewTableConfig, new_table_config: NewTableConfig,
namespace_client_properties: HashMap<String, String>,
session: Option<Arc<lance::session::Session>>, session: Option<Arc<lance::session::Session>>,
) -> Result<Self> { ) -> Result<Self> {
let session = session.unwrap_or_else(|| Arc::new(lance::session::Session::default())); let session = session.unwrap_or_else(|| Arc::new(lance::session::Session::default()));
@@ -429,6 +484,15 @@ impl ListingDatabase {
Self::try_create_dir(path).context(CreateDirSnafu { path })?; Self::try_create_dir(path).context(CreateDirSnafu { path })?;
} }
let namespace_database = Self::connect_namespace_database(
path,
HashMap::new(),
namespace_client_properties,
read_consistency_interval,
session.clone(),
)
.await?;
Ok(Self { Ok(Self {
uri: path.to_string(), uri: path.to_string(),
query_string: None, query_string: None,
@@ -440,6 +504,7 @@ impl ListingDatabase {
storage_options_provider: None, storage_options_provider: None,
new_table_config, new_table_config,
session, session,
namespace_database,
}) })
} }
@@ -497,6 +562,10 @@ impl ListingDatabase {
Ok(uri) Ok(uri)
} }
fn namespace_database(&self) -> Arc<LanceNamespaceDatabase> {
self.namespace_database.clone()
}
async fn drop_tables(&self, names: Vec<String>) -> Result<()> { async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
let object_store_params = ObjectStoreParams { let object_store_params = ObjectStoreParams {
storage_options_accessor: if self.storage_options.is_empty() { storage_options_accessor: if self.storage_options.is_empty() {
@@ -696,16 +765,7 @@ impl Database for ListingDatabase {
&self, &self,
request: ListNamespacesRequest, request: ListNamespacesRequest,
) -> Result<ListNamespacesResponse> { ) -> Result<ListNamespacesResponse> {
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) { self.namespace_database().list_namespaces(request).await
return Err(Error::NotSupported {
message: "Namespace operations are not supported for listing database".into(),
});
}
Ok(ListNamespacesResponse {
namespaces: Vec::new(),
page_token: None,
})
} }
fn uri(&self) -> &str { fn uri(&self) -> &str {
@@ -726,36 +786,26 @@ impl Database for ListingDatabase {
async fn create_namespace( async fn create_namespace(
&self, &self,
_request: CreateNamespaceRequest, request: CreateNamespaceRequest,
) -> Result<CreateNamespaceResponse> { ) -> Result<CreateNamespaceResponse> {
Err(Error::NotSupported { self.namespace_database().create_namespace(request).await
message: "Namespace operations are not supported for listing database".into(),
})
} }
async fn drop_namespace( async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
&self, self.namespace_database().drop_namespace(request).await
_request: DropNamespaceRequest,
) -> Result<DropNamespaceResponse> {
Err(Error::NotSupported {
message: "Namespace operations are not supported for listing database".into(),
})
} }
async fn describe_namespace( async fn describe_namespace(
&self, &self,
_request: DescribeNamespaceRequest, request: DescribeNamespaceRequest,
) -> Result<DescribeNamespaceResponse> { ) -> Result<DescribeNamespaceResponse> {
Err(Error::NotSupported { self.namespace_database().describe_namespace(request).await
message: "Namespace operations are not supported for listing database".into(),
})
} }
#[allow(deprecated)]
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> { async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
if !request.namespace_path.is_empty() { if !request.namespace_path.is_empty() {
return Err(Error::NotSupported { return self.namespace_database().table_names(request).await;
message: "Namespace parameter is not supported for listing database. Only root namespace is supported.".into(),
});
} }
let mut f = self let mut f = self
.object_store .object_store
@@ -788,9 +838,7 @@ impl Database for ListingDatabase {
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> { async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) { if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
return Err(Error::NotSupported { return self.namespace_database().list_tables(request).await;
message: "Namespace parameter is not supported for listing database. Only root namespace is supported.".into(),
});
} }
let mut f = self let mut f = self
.object_store .object_store
@@ -838,11 +886,8 @@ impl Database for ListingDatabase {
} }
async fn create_table(&self, request: CreateTableRequest) -> Result<Arc<dyn BaseTable>> { async fn create_table(&self, request: CreateTableRequest) -> Result<Arc<dyn BaseTable>> {
// When namespace is not empty, location must be provided if !request.namespace_path.is_empty() {
if !request.namespace_path.is_empty() && request.location.is_none() { return self.namespace_database().create_table(request).await;
return Err(Error::InvalidInput {
message: "Location must be provided when namespace is not empty".into(),
});
} }
// Use provided location if available, otherwise derive from table name // Use provided location if available, otherwise derive from table name
let table_uri = request let table_uri = request
@@ -959,11 +1004,8 @@ impl Database for ListingDatabase {
} }
async fn open_table(&self, mut request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> { async fn open_table(&self, mut request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
// When namespace is not empty, location must be provided if !request.namespace_path.is_empty() {
if !request.namespace_path.is_empty() && request.location.is_none() { return self.namespace_database().open_table(request).await;
return Err(Error::InvalidInput {
message: "Location must be provided when namespace is not empty".into(),
});
} }
// Use provided location if available, otherwise derive from table name // Use provided location if available, otherwise derive from table name
let table_uri = request let table_uri = request
@@ -1059,9 +1101,10 @@ impl Database for ListingDatabase {
async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> { async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> {
if !namespace_path.is_empty() { if !namespace_path.is_empty() {
return Err(Error::NotSupported { return self
message: "Namespace parameter is not supported for listing database.".into(), .namespace_database()
}); .drop_table(name, namespace_path)
.await;
} }
self.drop_tables(vec![name.to_string()]).await self.drop_tables(vec![name.to_string()]).await
} }
@@ -1070,9 +1113,10 @@ impl Database for ListingDatabase {
async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()> { async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()> {
// Check if namespace parameter is provided // Check if namespace parameter is provided
if !namespace_path.is_empty() { if !namespace_path.is_empty() {
return Err(Error::NotSupported { return self
message: "Namespace parameter is not supported for listing database.".into(), .namespace_database()
}); .drop_all_tables(namespace_path)
.await;
} }
let tables = self.table_names(TableNamesRequest::default()).await?; let tables = self.table_names(TableNamesRequest::default()).await?;
self.drop_tables(tables).await self.drop_tables(tables).await
@@ -1083,30 +1127,11 @@ impl Database for ListingDatabase {
} }
async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> { async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> {
// Create a DirectoryNamespace pointing to the same root with the same storage options self.namespace_database.namespace_client().await
let mut builder = lance_namespace_impls::DirectoryNamespaceBuilder::new(&self.uri);
// Add storage options
if !self.storage_options.is_empty() {
builder = builder.storage_options(self.storage_options.clone());
}
// Use the same session
builder = builder.session(self.session.clone());
let namespace = builder.build().await.map_err(|e| Error::Runtime {
message: format!("Failed to create namespace client: {}", e),
})?;
Ok(Arc::new(namespace) as Arc<dyn lance_namespace::LanceNamespace>)
} }
async fn namespace_client_config(&self) -> Result<(String, HashMap<String, String>)> { async fn namespace_client_config(&self) -> Result<(String, HashMap<String, String>)> {
let mut properties = HashMap::new(); self.namespace_database.namespace_client_config().await
properties.insert("root".to_string(), self.uri.clone());
for (key, value) in &self.storage_options {
properties.insert(format!("storage.{}", key), value.clone());
}
Ok(("dir".to_string(), properties))
} }
} }
@@ -1132,6 +1157,7 @@ mod tests {
#[cfg(feature = "remote")] #[cfg(feature = "remote")]
client_config: Default::default(), client_config: Default::default(),
options: Default::default(), options: Default::default(),
namespace_client_properties: Default::default(),
read_consistency_interval: None, read_consistency_interval: None,
session: None, session: None,
}; };
@@ -1265,6 +1291,7 @@ mod tests {
#[cfg(feature = "remote")] #[cfg(feature = "remote")]
client_config: Default::default(), client_config: Default::default(),
options: options.clone(), options: options.clone(),
namespace_client_properties: Default::default(),
read_consistency_interval: None, read_consistency_interval: None,
session: None, session: None,
}; };
@@ -1799,6 +1826,7 @@ mod tests {
#[cfg(feature = "remote")] #[cfg(feature = "remote")]
client_config: Default::default(), client_config: Default::default(),
options, options,
namespace_client_properties: Default::default(),
read_consistency_interval: None, read_consistency_interval: None,
session: None, session: None,
}; };
@@ -1904,6 +1932,7 @@ mod tests {
#[cfg(feature = "remote")] #[cfg(feature = "remote")]
client_config: Default::default(), client_config: Default::default(),
options, options,
namespace_client_properties: Default::default(),
read_consistency_interval: None, read_consistency_interval: None,
session: None, session: None,
}; };
@@ -1975,6 +2004,7 @@ mod tests {
#[cfg(feature = "remote")] #[cfg(feature = "remote")]
client_config: Default::default(), client_config: Default::default(),
options, options,
namespace_client_properties: Default::default(),
read_consistency_interval: None, read_consistency_interval: None,
session: None, session: None,
}; };
@@ -2108,4 +2138,208 @@ mod tests {
assert!(tables.contains(&"table1".to_string())); assert!(tables.contains(&"table1".to_string()));
assert!(tables.contains(&"table2".to_string())); assert!(tables.contains(&"table2".to_string()));
} }
#[tokio::test]
async fn test_listing_database_namespace_operations() {
let (_tempdir, db) = setup_database().await;
db.create_namespace(CreateNamespaceRequest {
id: Some(vec!["parent".to_string()]),
..Default::default()
})
.await
.unwrap();
db.create_namespace(CreateNamespaceRequest {
id: Some(vec!["parent".to_string(), "child".to_string()]),
..Default::default()
})
.await
.unwrap();
let root_namespaces = db
.list_namespaces(ListNamespacesRequest {
id: Some(vec![]),
..Default::default()
})
.await
.unwrap();
assert!(root_namespaces.namespaces.contains(&"parent".to_string()));
let child_namespaces = db
.list_namespaces(ListNamespacesRequest {
id: Some(vec!["parent".to_string()]),
..Default::default()
})
.await
.unwrap();
assert!(child_namespaces.namespaces.contains(&"child".to_string()));
db.describe_namespace(DescribeNamespaceRequest {
id: Some(vec!["parent".to_string(), "child".to_string()]),
..Default::default()
})
.await
.unwrap();
}
#[tokio::test]
async fn test_listing_database_with_namespace_client_properties() {
let tempdir = tempdir().unwrap();
let uri = tempdir.path().to_str().unwrap();
let mut namespace_client_properties = HashMap::new();
namespace_client_properties.insert(
"table_version_tracking_enabled".to_string(),
"true".to_string(),
);
namespace_client_properties.insert("manifest_enabled".to_string(), "true".to_string());
let request = ConnectRequest {
uri: uri.to_string(),
#[cfg(feature = "remote")]
client_config: Default::default(),
options: Default::default(),
namespace_client_properties,
read_consistency_interval: None,
session: None,
};
let db = ListingDatabase::connect_with_options(&request)
.await
.unwrap();
let namespace_path = vec!["test_ns".to_string()];
db.create_namespace(CreateNamespaceRequest {
id: Some(namespace_path.clone()),
..Default::default()
})
.await
.unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("name", DataType::Utf8, false),
]));
db.create_table(CreateTableRequest {
name: "managed_table".to_string(),
namespace_path: namespace_path.clone(),
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
})
.await
.unwrap();
let namespace_client = db.namespace_client().await.unwrap();
let describe = namespace_client
.describe_table(lance_namespace::models::DescribeTableRequest {
id: Some(vec!["test_ns".to_string(), "managed_table".to_string()]),
..Default::default()
})
.await
.unwrap();
assert_eq!(describe.managed_versioning, Some(true));
}
#[tokio::test]
async fn test_listing_database_nested_namespace_table_ops() {
let (_tempdir, db) = setup_database().await;
let namespace_path = vec!["parent".to_string(), "child".to_string()];
db.create_namespace(CreateNamespaceRequest {
id: Some(vec!["parent".to_string()]),
..Default::default()
})
.await
.unwrap();
db.create_namespace(CreateNamespaceRequest {
id: Some(namespace_path.clone()),
..Default::default()
})
.await
.unwrap();
let schema = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int32, false),
Field::new("name", DataType::Utf8, false),
]));
db.create_table(CreateTableRequest {
name: "nested_table".to_string(),
namespace_path: namespace_path.clone(),
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
mode: CreateTableMode::Create,
write_options: Default::default(),
location: None,
namespace_client: None,
})
.await
.unwrap();
let namespace_client = db.namespace_client().await.unwrap();
let describe = namespace_client
.describe_table(lance_namespace::models::DescribeTableRequest {
id: Some(vec![
"parent".to_string(),
"child".to_string(),
"nested_table".to_string(),
]),
..Default::default()
})
.await
.unwrap();
assert!(describe.location.is_some());
let table = db
.open_table(OpenTableRequest {
name: "nested_table".to_string(),
namespace_path: namespace_path.clone(),
index_cache_size: None,
lance_read_params: None,
location: None,
namespace_client: None,
managed_versioning: None,
})
.await
.unwrap();
assert_eq!(table.name(), "nested_table");
#[allow(deprecated)]
let table_names = db
.table_names(TableNamesRequest {
namespace_path: namespace_path.clone(),
start_after: None,
limit: None,
})
.await
.unwrap();
assert_eq!(table_names, vec!["nested_table".to_string()]);
let list_tables = db
.list_tables(ListTablesRequest {
id: Some(namespace_path.clone()),
..Default::default()
})
.await
.unwrap();
assert_eq!(list_tables.tables, vec!["nested_table".to_string()]);
db.drop_table("nested_table", &namespace_path)
.await
.unwrap();
let post_drop = db
.list_tables(ListTablesRequest {
id: Some(namespace_path),
..Default::default()
})
.await
.unwrap();
assert!(post_drop.tables.is_empty());
}
} }
+41
View File
@@ -450,6 +450,47 @@ mod tests {
)); ));
} }
#[tokio::test]
async fn test_namespace_connection_with_namespace_client_properties() {
let tmp_dir = tempdir().unwrap();
let root_path = tmp_dir.path().to_str().unwrap().to_string();
let mut properties = HashMap::new();
properties.insert("root".to_string(), root_path);
let conn = connect_namespace("dir", properties)
.namespace_client_property("table_version_tracking_enabled", "true")
.namespace_client_property("manifest_enabled", "true")
.execute()
.await
.expect("Failed to connect to namespace");
conn.create_namespace(CreateNamespaceRequest {
id: Some(vec!["test_ns".into()]),
..Default::default()
})
.await
.expect("Failed to create namespace");
let test_data = create_test_data();
conn.create_table("test_table", test_data)
.namespace(vec!["test_ns".into()])
.execute()
.await
.expect("Failed to create table");
let namespace_client = conn.namespace_client().await.unwrap();
let describe = namespace_client
.describe_table(DescribeTableRequest {
id: Some(vec!["test_ns".into(), "test_table".into()]),
..Default::default()
})
.await
.expect("Failed to describe table");
assert_eq!(describe.managed_versioning, Some(true));
}
#[tokio::test] #[tokio::test]
async fn test_namespace_create_table_basic() { async fn test_namespace_create_table_basic() {
// Setup: Create a temporary directory for the namespace // Setup: Create a temporary directory for the namespace
+1 -1
View File
@@ -69,7 +69,7 @@
//! It treats [`FixedSizeList<Float16/Float32>`](https://docs.rs/arrow/latest/arrow/array/struct.FixedSizeListArray.html) //! It treats [`FixedSizeList<Float16/Float32>`](https://docs.rs/arrow/latest/arrow/array/struct.FixedSizeListArray.html)
//! columns as vector columns. //! columns as vector columns.
//! //!
//! For more details, please refer to the [LanceDB documentation](https://lancedb.com/docs). //! For more details, please refer to the [LanceDB documentation](https://docs.lancedb.com).
//! //!
//! #### Create a table //! #### Create a table
//! //!
+1 -1
View File
@@ -97,7 +97,7 @@ pub struct RemoteDatabaseOptions {
pub host_override: Option<String>, pub host_override: Option<String>,
/// Storage options configure the storage layer (e.g. S3, GCS, Azure, etc.) /// Storage options configure the storage layer (e.g. S3, GCS, Azure, etc.)
/// ///
/// See available options at <https://lancedb.com/docs/storage/> /// See available options at <https://docs.lancedb.com/storage/>
/// ///
/// These options are only used for LanceDB Enterprise and only a subset of options /// These options are only used for LanceDB Enterprise and only a subset of options
/// are supported. /// are supported.