mirror of
https://github.com/lancedb/lancedb.git
synced 2026-06-13 17:20:38 +00:00
Compare commits
20 Commits
python-v0.
...
nested-rus
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
333b0c7c7c | ||
|
|
fd98b845ea | ||
|
|
be48ada352 | ||
|
|
9ad2dfe601 | ||
|
|
f909df3e87 | ||
|
|
d715bbb588 | ||
|
|
5ce3d8d141 | ||
|
|
5eaac178b1 | ||
|
|
11af763fcd | ||
|
|
2ed5452e1c | ||
|
|
b7c0b5987c | ||
|
|
97a4b38f19 | ||
|
|
10879d99b8 | ||
|
|
4e6a1d5dce | ||
|
|
13d2759356 | ||
|
|
7f52ec8c36 | ||
|
|
c6ae0de3ee | ||
|
|
231f0655ce | ||
|
|
8c52977c59 | ||
|
|
359710a0bf |
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.28.0-beta.2"
|
current_version = "0.28.0-beta.8"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
2
.github/ISSUE_TEMPLATE/documentation.yml
vendored
2
.github/ISSUE_TEMPLATE/documentation.yml
vendored
@@ -18,6 +18,6 @@ body:
|
|||||||
label: Link
|
label: Link
|
||||||
description: >
|
description: >
|
||||||
Provide a link to the existing documentation, if applicable.
|
Provide a link to the existing documentation, if applicable.
|
||||||
placeholder: ex. https://lancedb.com/docs/tables/...
|
placeholder: ex. https://docs.lancedb.com/tables/...
|
||||||
validations:
|
validations:
|
||||||
required: false
|
required: false
|
||||||
|
|||||||
386
Cargo.lock
generated
386
Cargo.lock
generated
@@ -137,15 +137,6 @@ dependencies = [
|
|||||||
"object",
|
"object",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "arc-swap"
|
|
||||||
version = "1.9.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
|
|
||||||
dependencies = [
|
|
||||||
"rustversion",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "argminmax"
|
name = "argminmax"
|
||||||
version = "0.6.3"
|
version = "0.6.3"
|
||||||
@@ -302,7 +293,7 @@ dependencies = [
|
|||||||
"arrow-schema",
|
"arrow-schema",
|
||||||
"arrow-select",
|
"arrow-select",
|
||||||
"flatbuffers",
|
"flatbuffers",
|
||||||
"lz4_flex 0.12.1",
|
"lz4_flex",
|
||||||
"zstd",
|
"zstd",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -1278,31 +1269,6 @@ dependencies = [
|
|||||||
"generic-array",
|
"generic-array",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bon"
|
|
||||||
version = "3.9.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
|
|
||||||
dependencies = [
|
|
||||||
"bon-macros",
|
|
||||||
"rustversion",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bon-macros"
|
|
||||||
version = "3.9.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
|
|
||||||
dependencies = [
|
|
||||||
"darling 0.20.11",
|
|
||||||
"ident_case",
|
|
||||||
"prettyplease",
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"rustversion",
|
|
||||||
"syn 2.0.117",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "brotli"
|
name = "brotli"
|
||||||
version = "3.5.0"
|
version = "3.5.0"
|
||||||
@@ -1472,12 +1438,6 @@ dependencies = [
|
|||||||
"shlex",
|
"shlex",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "census"
|
|
||||||
version = "0.4.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "cfg-if"
|
name = "cfg-if"
|
||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
@@ -1795,7 +1755,7 @@ dependencies = [
|
|||||||
"crossterm_winapi",
|
"crossterm_winapi",
|
||||||
"document-features",
|
"document-features",
|
||||||
"parking_lot",
|
"parking_lot",
|
||||||
"rustix 1.1.4",
|
"rustix",
|
||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2714,12 +2674,6 @@ dependencies = [
|
|||||||
"litrs",
|
"litrs",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "downcast-rs"
|
|
||||||
version = "2.0.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "dtor"
|
name = "dtor"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
@@ -2955,12 +2909,6 @@ version = "0.2.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
|
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fastdivide"
|
|
||||||
version = "0.4.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fastrand"
|
name = "fastrand"
|
||||||
version = "2.3.0"
|
version = "2.3.0"
|
||||||
@@ -3054,16 +3002,6 @@ dependencies = [
|
|||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fs4"
|
|
||||||
version = "0.8.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
|
|
||||||
dependencies = [
|
|
||||||
"rustix 0.38.44",
|
|
||||||
"windows-sys 0.52.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fs_extra"
|
name = "fs_extra"
|
||||||
version = "1.3.0"
|
version = "1.3.0"
|
||||||
@@ -3072,8 +3010,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fsst"
|
name = "fsst"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"rand 0.9.2",
|
"rand 0.9.2",
|
||||||
@@ -3560,12 +3498,6 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "htmlescape"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "http"
|
name = "http"
|
||||||
version = "0.2.12"
|
version = "0.2.12"
|
||||||
@@ -4134,8 +4066,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance"
|
name = "lance"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4178,12 +4110,14 @@ dependencies = [
|
|||||||
"lance-linalg",
|
"lance-linalg",
|
||||||
"lance-namespace",
|
"lance-namespace",
|
||||||
"lance-table",
|
"lance-table",
|
||||||
|
"lance-tokenizer",
|
||||||
"log",
|
"log",
|
||||||
"moka",
|
"moka",
|
||||||
"object_store",
|
"object_store",
|
||||||
"permutation",
|
"permutation",
|
||||||
"pin-project",
|
"pin-project",
|
||||||
"prost",
|
"prost",
|
||||||
|
"prost-build",
|
||||||
"prost-types",
|
"prost-types",
|
||||||
"rand 0.9.2",
|
"rand 0.9.2",
|
||||||
"roaring",
|
"roaring",
|
||||||
@@ -4191,7 +4125,6 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"snafu 0.9.0",
|
"snafu 0.9.0",
|
||||||
"tantivy",
|
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
@@ -4202,8 +4135,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-arrow"
|
name = "lance-arrow"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4224,8 +4157,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-bitpacking"
|
name = "lance-bitpacking"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrayref",
|
"arrayref",
|
||||||
"paste",
|
"paste",
|
||||||
@@ -4234,8 +4167,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-core"
|
name = "lance-core"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4272,8 +4205,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datafusion"
|
name = "lance-datafusion"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4304,8 +4237,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-datagen"
|
name = "lance-datagen"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4323,8 +4256,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-encoding"
|
name = "lance-encoding"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4361,8 +4294,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-file"
|
name = "lance-file"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4394,8 +4327,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-index"
|
name = "lance-index"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4432,6 +4365,7 @@ dependencies = [
|
|||||||
"lance-io",
|
"lance-io",
|
||||||
"lance-linalg",
|
"lance-linalg",
|
||||||
"lance-table",
|
"lance-table",
|
||||||
|
"lance-tokenizer",
|
||||||
"libm",
|
"libm",
|
||||||
"log",
|
"log",
|
||||||
"ndarray",
|
"ndarray",
|
||||||
@@ -4449,7 +4383,6 @@ dependencies = [
|
|||||||
"serde_json",
|
"serde_json",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"snafu 0.9.0",
|
"snafu 0.9.0",
|
||||||
"tantivy",
|
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
@@ -4459,8 +4392,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-io"
|
name = "lance-io"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-arith",
|
"arrow-arith",
|
||||||
@@ -4504,8 +4437,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-linalg"
|
name = "lance-linalg"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4521,8 +4454,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace"
|
name = "lance-namespace"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -4535,8 +4468,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-namespace-impls"
|
name = "lance-namespace-impls"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-ipc",
|
"arrow-ipc",
|
||||||
@@ -4581,8 +4514,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-table"
|
name = "lance-table"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
@@ -4621,8 +4554,8 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lance-testing"
|
name = "lance-testing"
|
||||||
version = "5.1.0-beta.2"
|
version = "6.0.0-beta.1"
|
||||||
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.2#df61d95cac9ab579e4bc4ff41d1bd749b24af7f0"
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-schema",
|
"arrow-schema",
|
||||||
@@ -4631,9 +4564,19 @@ dependencies = [
|
|||||||
"rand 0.9.2",
|
"rand 0.9.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "lance-tokenizer"
|
||||||
|
version = "6.0.0-beta.1"
|
||||||
|
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
|
||||||
|
dependencies = [
|
||||||
|
"rust-stemmers",
|
||||||
|
"serde",
|
||||||
|
"unicode-normalization",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.28.0-beta.2"
|
version = "0.28.0-beta.8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"ahash",
|
"ahash",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
@@ -4715,7 +4658,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
version = "0.28.0-beta.2"
|
version = "0.28.0-beta.8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow-array",
|
"arrow-array",
|
||||||
"arrow-buffer",
|
"arrow-buffer",
|
||||||
@@ -4737,7 +4680,7 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.31.0-beta.2"
|
version = "0.31.0-beta.8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arrow",
|
"arrow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -4775,12 +4718,6 @@ version = "0.1.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "levenshtein_automata"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lexical-core"
|
name = "lexical-core"
|
||||||
version = "1.0.6"
|
version = "1.0.6"
|
||||||
@@ -4869,12 +4806,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "linux-raw-sys"
|
|
||||||
version = "0.4.15"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linux-raw-sys"
|
name = "linux-raw-sys"
|
||||||
version = "0.12.1"
|
version = "0.12.1"
|
||||||
@@ -4955,12 +4886,6 @@ dependencies = [
|
|||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "lz4_flex"
|
|
||||||
version = "0.11.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lz4_flex"
|
name = "lz4_flex"
|
||||||
version = "0.12.1"
|
version = "0.12.1"
|
||||||
@@ -5035,15 +4960,6 @@ dependencies = [
|
|||||||
"digest",
|
"digest",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "measure_time"
|
|
||||||
version = "0.9.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
|
|
||||||
dependencies = [
|
|
||||||
"log",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "memchr"
|
name = "memchr"
|
||||||
version = "2.8.0"
|
version = "2.8.0"
|
||||||
@@ -5197,12 +5113,6 @@ dependencies = [
|
|||||||
"target-features",
|
"target-features",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "murmurhash32"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "napi"
|
name = "napi"
|
||||||
version = "3.8.3"
|
version = "3.8.3"
|
||||||
@@ -5491,12 +5401,6 @@ version = "1.70.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "oneshot"
|
|
||||||
version = "0.1.13"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "onig"
|
name = "onig"
|
||||||
version = "6.5.1"
|
version = "6.5.1"
|
||||||
@@ -5586,15 +5490,6 @@ version = "0.5.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "ownedbytes"
|
|
||||||
version = "0.9.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
|
|
||||||
dependencies = [
|
|
||||||
"stable_deref_trait",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "p256"
|
name = "p256"
|
||||||
version = "0.11.1"
|
version = "0.11.1"
|
||||||
@@ -7049,19 +6944,6 @@ dependencies = [
|
|||||||
"semver",
|
"semver",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rustix"
|
|
||||||
version = "0.38.44"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"errno",
|
|
||||||
"libc",
|
|
||||||
"linux-raw-sys 0.4.15",
|
|
||||||
"windows-sys 0.59.0",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustix"
|
name = "rustix"
|
||||||
version = "1.1.4"
|
version = "1.1.4"
|
||||||
@@ -7071,7 +6953,7 @@ dependencies = [
|
|||||||
"bitflags",
|
"bitflags",
|
||||||
"errno",
|
"errno",
|
||||||
"libc",
|
"libc",
|
||||||
"linux-raw-sys 0.12.1",
|
"linux-raw-sys",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -7536,15 +7418,6 @@ version = "1.0.2"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "sketches-ddsketch"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "slab"
|
name = "slab"
|
||||||
version = "0.4.12"
|
version = "0.4.12"
|
||||||
@@ -7917,152 +7790,6 @@ version = "0.2.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy"
|
|
||||||
version = "0.24.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43"
|
|
||||||
dependencies = [
|
|
||||||
"aho-corasick",
|
|
||||||
"arc-swap",
|
|
||||||
"base64 0.22.1",
|
|
||||||
"bitpacking",
|
|
||||||
"bon",
|
|
||||||
"byteorder",
|
|
||||||
"census",
|
|
||||||
"crc32fast",
|
|
||||||
"crossbeam-channel",
|
|
||||||
"downcast-rs",
|
|
||||||
"fastdivide",
|
|
||||||
"fnv",
|
|
||||||
"fs4",
|
|
||||||
"htmlescape",
|
|
||||||
"hyperloglogplus",
|
|
||||||
"itertools 0.14.0",
|
|
||||||
"levenshtein_automata",
|
|
||||||
"log",
|
|
||||||
"lru",
|
|
||||||
"lz4_flex 0.11.6",
|
|
||||||
"measure_time",
|
|
||||||
"memmap2 0.9.10",
|
|
||||||
"once_cell",
|
|
||||||
"oneshot",
|
|
||||||
"rayon",
|
|
||||||
"regex",
|
|
||||||
"rust-stemmers",
|
|
||||||
"rustc-hash",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"sketches-ddsketch",
|
|
||||||
"smallvec",
|
|
||||||
"tantivy-bitpacker",
|
|
||||||
"tantivy-columnar",
|
|
||||||
"tantivy-common",
|
|
||||||
"tantivy-fst",
|
|
||||||
"tantivy-query-grammar",
|
|
||||||
"tantivy-stacker",
|
|
||||||
"tantivy-tokenizer-api",
|
|
||||||
"tempfile",
|
|
||||||
"thiserror 2.0.18",
|
|
||||||
"time",
|
|
||||||
"uuid",
|
|
||||||
"winapi",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-bitpacker"
|
|
||||||
version = "0.8.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494"
|
|
||||||
dependencies = [
|
|
||||||
"bitpacking",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-columnar"
|
|
||||||
version = "0.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344"
|
|
||||||
dependencies = [
|
|
||||||
"downcast-rs",
|
|
||||||
"fastdivide",
|
|
||||||
"itertools 0.14.0",
|
|
||||||
"serde",
|
|
||||||
"tantivy-bitpacker",
|
|
||||||
"tantivy-common",
|
|
||||||
"tantivy-sstable",
|
|
||||||
"tantivy-stacker",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-common"
|
|
||||||
version = "0.9.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f"
|
|
||||||
dependencies = [
|
|
||||||
"async-trait",
|
|
||||||
"byteorder",
|
|
||||||
"ownedbytes",
|
|
||||||
"serde",
|
|
||||||
"time",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-fst"
|
|
||||||
version = "0.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
|
|
||||||
dependencies = [
|
|
||||||
"byteorder",
|
|
||||||
"regex-syntax",
|
|
||||||
"utf8-ranges",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-query-grammar"
|
|
||||||
version = "0.24.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a"
|
|
||||||
dependencies = [
|
|
||||||
"nom 7.1.3",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-sstable"
|
|
||||||
version = "0.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416"
|
|
||||||
dependencies = [
|
|
||||||
"futures-util",
|
|
||||||
"itertools 0.14.0",
|
|
||||||
"tantivy-bitpacker",
|
|
||||||
"tantivy-common",
|
|
||||||
"tantivy-fst",
|
|
||||||
"zstd",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-stacker"
|
|
||||||
version = "0.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1"
|
|
||||||
dependencies = [
|
|
||||||
"murmurhash32",
|
|
||||||
"rand_distr 0.4.3",
|
|
||||||
"tantivy-common",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tantivy-tokenizer-api"
|
|
||||||
version = "0.5.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tap"
|
name = "tap"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
@@ -8090,7 +7817,7 @@ dependencies = [
|
|||||||
"fastrand",
|
"fastrand",
|
||||||
"getrandom 0.4.2",
|
"getrandom 0.4.2",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"rustix 1.1.4",
|
"rustix",
|
||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -8542,6 +8269,15 @@ version = "1.0.24"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "unicode-normalization"
|
||||||
|
version = "0.1.25"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
|
||||||
|
dependencies = [
|
||||||
|
"tinyvec",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "unicode-normalization-alignments"
|
name = "unicode-normalization-alignments"
|
||||||
version = "0.1.12"
|
version = "0.1.12"
|
||||||
|
|||||||
28
Cargo.toml
28
Cargo.toml
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
|
|||||||
rust-version = "1.91.0"
|
rust-version = "1.91.0"
|
||||||
|
|
||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
lance = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-core = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-core = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-datagen = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datagen = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-file = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-file = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-io = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-io = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-index = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-index = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-linalg = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-linalg = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-namespace = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-namespace-impls = { "version" = "=5.1.0-beta.2", default-features = false, "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-namespace-impls = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-table = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-table = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-testing = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-testing = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-datafusion = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-datafusion = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-encoding = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-encoding = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
lance-arrow = { "version" = "=5.1.0-beta.2", "tag" = "v5.1.0-beta.2", "git" = "https://github.com/lance-format/lance.git" }
|
lance-arrow = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
|
||||||
ahash = "0.8"
|
ahash = "0.8"
|
||||||
# Note that this one does not include pyarrow
|
# Note that this one does not include pyarrow
|
||||||
arrow = { version = "57.2", optional = false }
|
arrow = { version = "57.2", optional = false }
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
|
|
||||||
# **The Multimodal AI Lakehouse**
|
# **The Multimodal AI Lakehouse**
|
||||||
|
|
||||||
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://lancedb.com/docs) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
|
[**How to Install** ](#how-to-install) ✦ [**Detailed Documentation**](https://docs.lancedb.com) ✦ [**Tutorials and Recipes**](https://github.com/lancedb/vectordb-recipes/tree/main) ✦ [**Contributors**](#contributors)
|
||||||
|
|
||||||
**The ultimate multimodal data platform for AI/ML applications.**
|
**The ultimate multimodal data platform for AI/ML applications.**
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ LanceDB is a central location where developers can build, train and analyze thei
|
|||||||
|
|
||||||
## **How to Install**:
|
## **How to Install**:
|
||||||
|
|
||||||
Follow the [Quickstart](https://lancedb.com/docs/quickstart/) doc to set up LanceDB locally.
|
Follow the [Quickstart](https://docs.lancedb.com/quickstart) doc to set up LanceDB locally.
|
||||||
|
|
||||||
**API & SDK:** We also support Python, Typescript and Rust SDKs
|
**API & SDK:** We also support Python, Typescript and Rust SDKs
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# LanceDB Documentation
|
# LanceDB Documentation
|
||||||
|
|
||||||
LanceDB docs are available at [lancedb.com/docs](https://lancedb.com/docs).
|
LanceDB docs are available at [docs.lancedb.com](https://docs.lancedb.com).
|
||||||
|
|
||||||
The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
|
The SDK docs are built and deployed automatically by [Github Actions](../.github/workflows/docs.yml)
|
||||||
whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
|
whenever a commit is pushed to the `main` branch. So it is possible for the docs to show
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
|
|||||||
<dependency>
|
<dependency>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-core</artifactId>
|
<artifactId>lancedb-core</artifactId>
|
||||||
<version>0.28.0-beta.2</version>
|
<version>0.28.0-beta.8</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
|
|||||||
console.log(results);
|
console.log(results);
|
||||||
```
|
```
|
||||||
|
|
||||||
The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples.
|
The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
|
|||||||
@@ -89,4 +89,4 @@ optional storageOptions: Record<string, string>;
|
|||||||
|
|
||||||
(For LanceDB OSS only): configuration for object storage.
|
(For LanceDB OSS only): configuration for object storage.
|
||||||
|
|
||||||
The available options are described at https://lancedb.com/docs/storage/
|
The available options are described at https://docs.lancedb.com/storage/
|
||||||
|
|||||||
@@ -97,4 +97,4 @@ Configuration for object storage.
|
|||||||
Options already set on the connection will be inherited by the table,
|
Options already set on the connection will be inherited by the table,
|
||||||
but can be overridden here.
|
but can be overridden here.
|
||||||
|
|
||||||
The available options are described at https://lancedb.com/docs/storage/
|
The available options are described at https://docs.lancedb.com/storage/
|
||||||
|
|||||||
@@ -42,4 +42,4 @@ Configuration for object storage.
|
|||||||
Options already set on the connection will be inherited by the table,
|
Options already set on the connection will be inherited by the table,
|
||||||
but can be overridden here.
|
but can be overridden here.
|
||||||
|
|
||||||
The available options are described at https://lancedb.com/docs/storage/
|
The available options are described at https://docs.lancedb.com/storage/
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
<parent>
|
<parent>
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.28.0-beta.2</version>
|
<version>0.28.0-beta.8</version>
|
||||||
<relativePath>../pom.xml</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
<groupId>com.lancedb</groupId>
|
<groupId>com.lancedb</groupId>
|
||||||
<artifactId>lancedb-parent</artifactId>
|
<artifactId>lancedb-parent</artifactId>
|
||||||
<version>0.28.0-beta.2</version>
|
<version>0.28.0-beta.8</version>
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
<name>${project.artifactId}</name>
|
<name>${project.artifactId}</name>
|
||||||
<description>LanceDB Java SDK Parent POM</description>
|
<description>LanceDB Java SDK Parent POM</description>
|
||||||
@@ -28,7 +28,7 @@
|
|||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
<arrow.version>15.0.0</arrow.version>
|
<arrow.version>15.0.0</arrow.version>
|
||||||
<lance-core.version>5.1.0-beta.2</lance-core.version>
|
<lance-core.version>6.0.0-beta.1</lance-core.version>
|
||||||
<spotless.skip>false</spotless.skip>
|
<spotless.skip>false</spotless.skip>
|
||||||
<spotless.version>2.30.0</spotless.version>
|
<spotless.version>2.30.0</spotless.version>
|
||||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-nodejs"
|
name = "lancedb-nodejs"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
version = "0.28.0-beta.2"
|
version = "0.28.0-beta.8"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
description.workspace = true
|
description.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
|
|||||||
@@ -30,7 +30,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
|
|||||||
console.log(results);
|
console.log(results);
|
||||||
```
|
```
|
||||||
|
|
||||||
The [quickstart](https://lancedb.com/docs/quickstart/basic-usage/) contains more complete examples.
|
The [quickstart](https://docs.lancedb.com/quickstart/) contains more complete examples.
|
||||||
|
|
||||||
## Development
|
## Development
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
// SPDX-License-Identifier: Apache-2.0
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
|
import { spawn } from "node:child_process";
|
||||||
|
import * as path from "node:path";
|
||||||
import { RecordBatch } from "apache-arrow";
|
import { RecordBatch } from "apache-arrow";
|
||||||
import * as tmp from "tmp";
|
import * as tmp from "tmp";
|
||||||
import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
|
import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
|
||||||
@@ -76,4 +78,91 @@ describe("rerankers", function () {
|
|||||||
|
|
||||||
expect(result).toHaveLength(2);
|
expect(result).toHaveLength(2);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("does not keep process alive after rerank query", async function () {
|
||||||
|
const script = `
|
||||||
|
import * as lancedb from "./dist/index.js";
|
||||||
|
import * as os from "node:os";
|
||||||
|
import * as path from "node:path";
|
||||||
|
import * as fs from "node:fs/promises";
|
||||||
|
|
||||||
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "lancedb-rerank-exit-"));
|
||||||
|
const db = await lancedb.connect(dir);
|
||||||
|
const table = await db.createTable("test", [{ text: "hello", vector: [1, 2, 3] }], {
|
||||||
|
mode: "overwrite",
|
||||||
|
});
|
||||||
|
await table.createIndex("text", { config: lancedb.Index.fts() });
|
||||||
|
await table.waitForIndex(["text_idx"], 30);
|
||||||
|
|
||||||
|
const reranker = await lancedb.rerankers.RRFReranker.create();
|
||||||
|
await table
|
||||||
|
.query()
|
||||||
|
.nearestTo([1, 2, 3])
|
||||||
|
.fullTextSearch("hello")
|
||||||
|
.rerank(reranker)
|
||||||
|
.toArray();
|
||||||
|
|
||||||
|
table.close();
|
||||||
|
db.close();
|
||||||
|
`;
|
||||||
|
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
const child = spawn(
|
||||||
|
process.execPath,
|
||||||
|
["--input-type=module", "-e", script],
|
||||||
|
{
|
||||||
|
cwd: path.resolve(__dirname, ".."),
|
||||||
|
stdio: ["ignore", "pipe", "pipe"],
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
|
||||||
|
child.stdout.on("data", (chunk) => {
|
||||||
|
stdout += chunk.toString();
|
||||||
|
});
|
||||||
|
|
||||||
|
child.stderr.on("data", (chunk) => {
|
||||||
|
stderr += chunk.toString();
|
||||||
|
});
|
||||||
|
|
||||||
|
const timeout = setTimeout(() => {
|
||||||
|
child.kill();
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`child process did not exit in time\nstdout:\n${stdout}\nstderr:\n${stderr}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}, 20_000);
|
||||||
|
|
||||||
|
child.on("error", (err) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
|
||||||
|
child.on("exit", (code, signal) => {
|
||||||
|
clearTimeout(timeout);
|
||||||
|
if (signal !== null) {
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`child process exited with signal ${signal}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (code !== 0) {
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`child process exited with code ${code}\nstdout:\n${stdout}\nstderr:\n${stderr}`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ export interface CreateTableOptions {
|
|||||||
* Options already set on the connection will be inherited by the table,
|
* Options already set on the connection will be inherited by the table,
|
||||||
* but can be overridden here.
|
* but can be overridden here.
|
||||||
*
|
*
|
||||||
* The available options are described at https://lancedb.com/docs/storage/
|
* The available options are described at https://docs.lancedb.com/storage/
|
||||||
*/
|
*/
|
||||||
storageOptions?: Record<string, string>;
|
storageOptions?: Record<string, string>;
|
||||||
|
|
||||||
@@ -78,7 +78,7 @@ export interface OpenTableOptions {
|
|||||||
* Options already set on the connection will be inherited by the table,
|
* Options already set on the connection will be inherited by the table,
|
||||||
* but can be overridden here.
|
* but can be overridden here.
|
||||||
*
|
*
|
||||||
* The available options are described at https://lancedb.com/docs/storage/
|
* The available options are described at https://docs.lancedb.com/storage/
|
||||||
*/
|
*/
|
||||||
storageOptions?: Record<string, string>;
|
storageOptions?: Record<string, string>;
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-darwin-arm64",
|
"name": "@lancedb/lancedb-darwin-arm64",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": ["darwin"],
|
"os": ["darwin"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.darwin-arm64.node",
|
"main": "lancedb.darwin-arm64.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
"name": "@lancedb/lancedb-linux-arm64-gnu",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-gnu.node",
|
"main": "lancedb.linux-arm64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-arm64-musl",
|
"name": "@lancedb/lancedb-linux-arm64-musl",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["arm64"],
|
"cpu": ["arm64"],
|
||||||
"main": "lancedb.linux-arm64-musl.node",
|
"main": "lancedb.linux-arm64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-gnu",
|
"name": "@lancedb/lancedb-linux-x64-gnu",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-gnu.node",
|
"main": "lancedb.linux-x64-gnu.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-linux-x64-musl",
|
"name": "@lancedb/lancedb-linux-x64-musl",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": ["linux"],
|
"os": ["linux"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.linux-x64-musl.node",
|
"main": "lancedb.linux-x64-musl.node",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
"name": "@lancedb/lancedb-win32-arm64-msvc",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": [
|
"os": [
|
||||||
"win32"
|
"win32"
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb-win32-x64-msvc",
|
"name": "@lancedb/lancedb-win32-x64-msvc",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"os": ["win32"],
|
"os": ["win32"],
|
||||||
"cpu": ["x64"],
|
"cpu": ["x64"],
|
||||||
"main": "lancedb.win32-x64-msvc.node",
|
"main": "lancedb.win32-x64-msvc.node",
|
||||||
|
|||||||
4
nodejs/package-lock.json
generated
4
nodejs/package-lock.json
generated
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@lancedb/lancedb",
|
"name": "@lancedb/lancedb",
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"cpu": [
|
"cpu": [
|
||||||
"x64",
|
"x64",
|
||||||
"arm64"
|
"arm64"
|
||||||
|
|||||||
@@ -11,7 +11,7 @@
|
|||||||
"ann"
|
"ann"
|
||||||
],
|
],
|
||||||
"private": false,
|
"private": false,
|
||||||
"version": "0.28.0-beta.2",
|
"version": "0.28.0-beta.8",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"exports": {
|
"exports": {
|
||||||
".": "./dist/index.js",
|
".": "./dist/index.js",
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ pub struct ConnectionOptions {
|
|||||||
pub read_consistency_interval: Option<f64>,
|
pub read_consistency_interval: Option<f64>,
|
||||||
/// (For LanceDB OSS only): configuration for object storage.
|
/// (For LanceDB OSS only): configuration for object storage.
|
||||||
///
|
///
|
||||||
/// The available options are described at https://lancedb.com/docs/storage/
|
/// The available options are described at https://docs.lancedb.com/storage/
|
||||||
pub storage_options: Option<HashMap<String, String>>,
|
pub storage_options: Option<HashMap<String, String>>,
|
||||||
/// (For LanceDB OSS only): the session to use for this connection. Holds
|
/// (For LanceDB OSS only): the session to use for this connection. Holds
|
||||||
/// shared caches and other session-specific state.
|
/// shared caches and other session-specific state.
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ type RerankHybridFn = ThreadsafeFunction<
|
|||||||
RerankHybridCallbackArgs,
|
RerankHybridCallbackArgs,
|
||||||
Status,
|
Status,
|
||||||
false,
|
false,
|
||||||
|
true,
|
||||||
>;
|
>;
|
||||||
|
|
||||||
/// Reranker implementation that "wraps" a NodeJS Reranker implementation.
|
/// Reranker implementation that "wraps" a NodeJS Reranker implementation.
|
||||||
@@ -32,7 +33,10 @@ impl Reranker {
|
|||||||
pub fn new(
|
pub fn new(
|
||||||
rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
|
rerank_hybrid: Function<RerankHybridCallbackArgs, Promise<Buffer>>,
|
||||||
) -> napi::Result<Self> {
|
) -> napi::Result<Self> {
|
||||||
let rerank_hybrid = rerank_hybrid.build_threadsafe_function().build()?;
|
let rerank_hybrid = rerank_hybrid
|
||||||
|
.build_threadsafe_function()
|
||||||
|
.weak::<true>()
|
||||||
|
.build()?;
|
||||||
Ok(Self { rerank_hybrid })
|
Ok(Self { rerank_hybrid })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
[tool.bumpversion]
|
[tool.bumpversion]
|
||||||
current_version = "0.31.0-beta.3"
|
current_version = "0.31.0-beta.8"
|
||||||
parse = """(?x)
|
parse = """(?x)
|
||||||
(?P<major>0|[1-9]\\d*)\\.
|
(?P<major>0|[1-9]\\d*)\\.
|
||||||
(?P<minor>0|[1-9]\\d*)\\.
|
(?P<minor>0|[1-9]\\d*)\\.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb-python"
|
name = "lancedb-python"
|
||||||
version = "0.31.0-beta.3"
|
version = "0.31.0-beta.8"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "Python bindings for LanceDB"
|
description = "Python bindings for LanceDB"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -110,7 +110,7 @@ def connect(
|
|||||||
default configuration is used.
|
default configuration is used.
|
||||||
storage_options: dict, optional
|
storage_options: dict, optional
|
||||||
Additional options for the storage backend. See available options at
|
Additional options for the storage backend. See available options at
|
||||||
<https://lancedb.com/docs/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
session: Session, optional
|
session: Session, optional
|
||||||
(For LanceDB OSS only)
|
(For LanceDB OSS only)
|
||||||
A session to use for this connection. Sessions allow you to configure
|
A session to use for this connection. Sessions allow you to configure
|
||||||
@@ -215,6 +215,85 @@ def connect(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
WORKER_PROPERTY_PREFIX = "_lancedb_worker_"
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_worker_overrides(props: dict[str, str]) -> dict[str, str]:
|
||||||
|
"""Apply worker property overrides.
|
||||||
|
|
||||||
|
Any key starting with ``_lancedb_worker_`` is extracted, the prefix
|
||||||
|
is stripped, and the resulting key-value pair is put back into the
|
||||||
|
map (overriding the existing value if present). The original
|
||||||
|
prefixed key is removed.
|
||||||
|
"""
|
||||||
|
worker_keys = [k for k in props if k.startswith(WORKER_PROPERTY_PREFIX)]
|
||||||
|
if not worker_keys:
|
||||||
|
return props
|
||||||
|
result = dict(props)
|
||||||
|
for key in worker_keys:
|
||||||
|
value = result.pop(key)
|
||||||
|
real_key = key[len(WORKER_PROPERTY_PREFIX) :]
|
||||||
|
result[real_key] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def deserialize_conn(
|
||||||
|
data: str,
|
||||||
|
*,
|
||||||
|
for_worker: bool = False,
|
||||||
|
) -> DBConnection:
|
||||||
|
"""Reconstruct a DBConnection from a serialized string.
|
||||||
|
|
||||||
|
The string must have been produced by
|
||||||
|
:meth:`DBConnection.serialize`.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
data : str
|
||||||
|
String produced by ``serialize()``.
|
||||||
|
for_worker : bool, default False
|
||||||
|
When ``True``, any namespace client property whose key starts
|
||||||
|
with ``_lancedb_worker_`` has that prefix stripped and the
|
||||||
|
value overrides the corresponding property. For example,
|
||||||
|
``_lancedb_worker_uri`` replaces ``uri``.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
DBConnection
|
||||||
|
A new connection matching the serialized state.
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
|
||||||
|
parsed = json.loads(data)
|
||||||
|
connection_type = parsed.get("connection_type")
|
||||||
|
|
||||||
|
rci_secs = parsed.get("read_consistency_interval_seconds")
|
||||||
|
rci = timedelta(seconds=rci_secs) if rci_secs is not None else None
|
||||||
|
storage_options = parsed.get("storage_options")
|
||||||
|
|
||||||
|
if connection_type == "namespace":
|
||||||
|
props = dict(parsed.get("namespace_client_properties") or {})
|
||||||
|
if for_worker:
|
||||||
|
props = _apply_worker_overrides(props)
|
||||||
|
return connect_namespace(
|
||||||
|
namespace_client_impl=parsed["namespace_client_impl"],
|
||||||
|
namespace_client_properties=props,
|
||||||
|
read_consistency_interval=rci,
|
||||||
|
storage_options=storage_options,
|
||||||
|
namespace_client_pushdown_operations=parsed.get(
|
||||||
|
"namespace_client_pushdown_operations"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
elif connection_type == "local":
|
||||||
|
return LanceDBConnection(
|
||||||
|
parsed["uri"],
|
||||||
|
read_consistency_interval=rci,
|
||||||
|
storage_options=storage_options,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unknown connection_type: {connection_type}")
|
||||||
|
|
||||||
|
|
||||||
async def connect_async(
|
async def connect_async(
|
||||||
uri: URI,
|
uri: URI,
|
||||||
*,
|
*,
|
||||||
@@ -257,7 +336,7 @@ async def connect_async(
|
|||||||
default configuration is used.
|
default configuration is used.
|
||||||
storage_options: dict, optional
|
storage_options: dict, optional
|
||||||
Additional options for the storage backend. See available options at
|
Additional options for the storage backend. See available options at
|
||||||
<https://lancedb.com/docs/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
session: Session, optional
|
session: Session, optional
|
||||||
(For LanceDB OSS only)
|
(For LanceDB OSS only)
|
||||||
A session to use for this connection. Sessions allow you to configure
|
A session to use for this connection. Sessions allow you to configure
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ def data_to_reader(
|
|||||||
f"Unknown data type {type(data)}. "
|
f"Unknown data type {type(data)}. "
|
||||||
"Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
|
"Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
|
||||||
"pyarrow Table/RecordBatch, or Pydantic models. "
|
"pyarrow Table/RecordBatch, or Pydantic models. "
|
||||||
"See https://lancedb.com/docs/tables/ for examples."
|
"See https://docs.lancedb.com/tables/ for examples."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -282,7 +282,7 @@ class DBConnection(EnforceOverrides):
|
|||||||
Additional options for the storage backend. Options already set on the
|
Additional options for the storage backend. Options already set on the
|
||||||
connection will be inherited by the table, but can be overridden here.
|
connection will be inherited by the table, but can be overridden here.
|
||||||
See available options at
|
See available options at
|
||||||
<https://lancedb.com/docs/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
|
|
||||||
To enable stable row IDs (row IDs remain stable after compaction,
|
To enable stable row IDs (row IDs remain stable after compaction,
|
||||||
update, delete, and merges), set `new_table_enable_stable_row_ids`
|
update, delete, and merges), set `new_table_enable_stable_row_ids`
|
||||||
@@ -433,7 +433,7 @@ class DBConnection(EnforceOverrides):
|
|||||||
Additional options for the storage backend. Options already set on the
|
Additional options for the storage backend. Options already set on the
|
||||||
connection will be inherited by the table, but can be overridden here.
|
connection will be inherited by the table, but can be overridden here.
|
||||||
See available options at
|
See available options at
|
||||||
<https://lancedb.com/docs/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
@@ -529,6 +529,19 @@ class DBConnection(EnforceOverrides):
|
|||||||
"namespace_client is not supported for this connection type"
|
"namespace_client is not supported for this connection type"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def serialize(self) -> str:
|
||||||
|
"""Serialize this connection for reconstruction.
|
||||||
|
|
||||||
|
The returned string can be passed to :func:`lancedb.deserialize_conn`
|
||||||
|
to recreate an equivalent connection, e.g. in a remote worker.
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
str
|
||||||
|
Serialized representation of this connection.
|
||||||
|
"""
|
||||||
|
raise NotImplementedError("serialize is not supported for this connection type")
|
||||||
|
|
||||||
|
|
||||||
class LanceDBConnection(DBConnection):
|
class LanceDBConnection(DBConnection):
|
||||||
"""
|
"""
|
||||||
@@ -581,6 +594,7 @@ class LanceDBConnection(DBConnection):
|
|||||||
):
|
):
|
||||||
if _inner is not None:
|
if _inner is not None:
|
||||||
self._conn = _inner
|
self._conn = _inner
|
||||||
|
self._cached_namespace_client = None
|
||||||
return
|
return
|
||||||
|
|
||||||
if not isinstance(uri, Path):
|
if not isinstance(uri, Path):
|
||||||
@@ -628,6 +642,7 @@ class LanceDBConnection(DBConnection):
|
|||||||
# beyond _conn.
|
# beyond _conn.
|
||||||
self.storage_options = storage_options
|
self.storage_options = storage_options
|
||||||
self._conn = AsyncConnection(LOOP.run(do_connect()))
|
self._conn = AsyncConnection(LOOP.run(do_connect()))
|
||||||
|
self._cached_namespace_client: Optional[LanceNamespace] = None
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def read_consistency_interval(self) -> Optional[timedelta]:
|
def read_consistency_interval(self) -> Optional[timedelta]:
|
||||||
@@ -652,6 +667,22 @@ class LanceDBConnection(DBConnection):
|
|||||||
val += ")"
|
val += ")"
|
||||||
return val
|
return val
|
||||||
|
|
||||||
|
@override
|
||||||
|
def serialize(self) -> str:
|
||||||
|
import json
|
||||||
|
|
||||||
|
rci = self.read_consistency_interval
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"connection_type": "local",
|
||||||
|
"uri": self.uri,
|
||||||
|
"storage_options": self.storage_options,
|
||||||
|
"read_consistency_interval_seconds": (
|
||||||
|
rci.total_seconds() if rci else None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
async def _async_get_table_names(self, start_after: Optional[str], limit: int):
|
async def _async_get_table_names(self, start_after: Optional[str], limit: int):
|
||||||
conn = AsyncConnection(await lancedb_connect(self.uri))
|
conn = AsyncConnection(await lancedb_connect(self.uri))
|
||||||
return await conn.table_names(start_after=start_after, limit=limit)
|
return await conn.table_names(start_after=start_after, limit=limit)
|
||||||
@@ -687,10 +718,10 @@ class LanceDBConnection(DBConnection):
|
|||||||
"""
|
"""
|
||||||
if namespace_path is None:
|
if namespace_path is None:
|
||||||
namespace_path = []
|
namespace_path = []
|
||||||
return LOOP.run(
|
return self._namespace_conn().list_namespaces(
|
||||||
self._conn.list_namespaces(
|
namespace_path=namespace_path,
|
||||||
namespace_path=namespace_path, page_token=page_token, limit=limit
|
page_token=page_token,
|
||||||
)
|
limit=limit,
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -700,27 +731,10 @@ class LanceDBConnection(DBConnection):
|
|||||||
mode: Optional[str] = None,
|
mode: Optional[str] = None,
|
||||||
properties: Optional[Dict[str, str]] = None,
|
properties: Optional[Dict[str, str]] = None,
|
||||||
) -> CreateNamespaceResponse:
|
) -> CreateNamespaceResponse:
|
||||||
"""Create a new namespace.
|
return self._namespace_conn().create_namespace(
|
||||||
|
namespace_path=namespace_path,
|
||||||
Parameters
|
mode=mode,
|
||||||
----------
|
properties=properties,
|
||||||
namespace_path: List[str]
|
|
||||||
The namespace identifier to create.
|
|
||||||
mode: str, optional
|
|
||||||
Creation mode - "create" (fail if exists), "exist_ok" (skip if exists),
|
|
||||||
or "overwrite" (replace if exists). Case insensitive.
|
|
||||||
properties: Dict[str, str], optional
|
|
||||||
Properties to set on the namespace.
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
CreateNamespaceResponse
|
|
||||||
Response containing the properties of the created namespace.
|
|
||||||
"""
|
|
||||||
return LOOP.run(
|
|
||||||
self._conn.create_namespace(
|
|
||||||
namespace_path=namespace_path, mode=mode, properties=properties
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -730,46 +744,19 @@ class LanceDBConnection(DBConnection):
|
|||||||
mode: Optional[str] = None,
|
mode: Optional[str] = None,
|
||||||
behavior: Optional[str] = None,
|
behavior: Optional[str] = None,
|
||||||
) -> DropNamespaceResponse:
|
) -> DropNamespaceResponse:
|
||||||
"""Drop a namespace.
|
return self._namespace_conn().drop_namespace(
|
||||||
|
namespace_path=namespace_path,
|
||||||
Parameters
|
mode=mode,
|
||||||
----------
|
behavior=behavior,
|
||||||
namespace_path: List[str]
|
|
||||||
The namespace identifier to drop.
|
|
||||||
mode: str, optional
|
|
||||||
Whether to skip if not exists ("SKIP") or fail ("FAIL"). Case insensitive.
|
|
||||||
behavior: str, optional
|
|
||||||
Whether to restrict drop if not empty ("RESTRICT") or cascade ("CASCADE").
|
|
||||||
Case insensitive.
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
DropNamespaceResponse
|
|
||||||
Response containing properties and transaction_id if applicable.
|
|
||||||
"""
|
|
||||||
return LOOP.run(
|
|
||||||
self._conn.drop_namespace(
|
|
||||||
namespace_path=namespace_path, mode=mode, behavior=behavior
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def describe_namespace(
|
def describe_namespace(
|
||||||
self, namespace_path: List[str]
|
self, namespace_path: List[str]
|
||||||
) -> DescribeNamespaceResponse:
|
) -> DescribeNamespaceResponse:
|
||||||
"""Describe a namespace.
|
return self._namespace_conn().describe_namespace(
|
||||||
|
namespace_path=namespace_path,
|
||||||
Parameters
|
)
|
||||||
----------
|
|
||||||
namespace_path: List[str]
|
|
||||||
The namespace identifier to describe.
|
|
||||||
|
|
||||||
Returns
|
|
||||||
-------
|
|
||||||
DescribeNamespaceResponse
|
|
||||||
Response containing the namespace properties.
|
|
||||||
"""
|
|
||||||
return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path))
|
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def list_tables(
|
def list_tables(
|
||||||
@@ -798,6 +785,12 @@ class LanceDBConnection(DBConnection):
|
|||||||
"""
|
"""
|
||||||
if namespace_path is None:
|
if namespace_path is None:
|
||||||
namespace_path = []
|
namespace_path = []
|
||||||
|
if namespace_path:
|
||||||
|
return self._namespace_conn().list_tables(
|
||||||
|
namespace_path=namespace_path,
|
||||||
|
page_token=page_token,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
return LOOP.run(
|
return LOOP.run(
|
||||||
self._conn.list_tables(
|
self._conn.list_tables(
|
||||||
namespace_path=namespace_path, page_token=page_token, limit=limit
|
namespace_path=namespace_path, page_token=page_token, limit=limit
|
||||||
@@ -886,6 +879,22 @@ class LanceDBConnection(DBConnection):
|
|||||||
raise ValueError("mode must be either 'create' or 'overwrite'")
|
raise ValueError("mode must be either 'create' or 'overwrite'")
|
||||||
validate_table_name(name)
|
validate_table_name(name)
|
||||||
|
|
||||||
|
if namespace_path:
|
||||||
|
return self._namespace_conn().create_table(
|
||||||
|
name,
|
||||||
|
data=data,
|
||||||
|
schema=schema,
|
||||||
|
mode=mode,
|
||||||
|
exist_ok=exist_ok,
|
||||||
|
on_bad_vectors=on_bad_vectors,
|
||||||
|
fill_value=fill_value,
|
||||||
|
embedding_functions=embedding_functions,
|
||||||
|
namespace_path=namespace_path,
|
||||||
|
storage_options=storage_options,
|
||||||
|
data_storage_version=data_storage_version,
|
||||||
|
enable_v2_manifest_paths=enable_v2_manifest_paths,
|
||||||
|
)
|
||||||
|
|
||||||
tbl = LanceTable.create(
|
tbl = LanceTable.create(
|
||||||
self,
|
self,
|
||||||
name,
|
name,
|
||||||
@@ -901,6 +910,19 @@ class LanceDBConnection(DBConnection):
|
|||||||
)
|
)
|
||||||
return tbl
|
return tbl
|
||||||
|
|
||||||
|
def _namespace_conn(self) -> DBConnection:
|
||||||
|
"""Return a LanceNamespaceDBConnection backed by this connection's
|
||||||
|
directory namespace. Used to delegate child-namespace operations."""
|
||||||
|
from lancedb.namespace import LanceNamespaceDBConnection
|
||||||
|
|
||||||
|
return LanceNamespaceDBConnection(
|
||||||
|
self.namespace_client(),
|
||||||
|
read_consistency_interval=self.read_consistency_interval,
|
||||||
|
storage_options=self.storage_options,
|
||||||
|
namespace_client_impl=None,
|
||||||
|
namespace_client_properties=None,
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def open_table(
|
def open_table(
|
||||||
self,
|
self,
|
||||||
@@ -917,7 +939,8 @@ class LanceDBConnection(DBConnection):
|
|||||||
name: str
|
name: str
|
||||||
The name of the table.
|
The name of the table.
|
||||||
namespace_path: List[str], optional
|
namespace_path: List[str], optional
|
||||||
The namespace to open the table from.
|
The namespace to open the table from. When non-empty, the
|
||||||
|
table is resolved through the directory namespace client.
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
@@ -936,6 +959,14 @@ class LanceDBConnection(DBConnection):
|
|||||||
stacklevel=2,
|
stacklevel=2,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if namespace_path:
|
||||||
|
return self._namespace_conn().open_table(
|
||||||
|
name,
|
||||||
|
namespace_path=namespace_path,
|
||||||
|
storage_options=storage_options,
|
||||||
|
index_cache_size=index_cache_size,
|
||||||
|
)
|
||||||
|
|
||||||
return LanceTable.open(
|
return LanceTable.open(
|
||||||
self,
|
self,
|
||||||
name,
|
name,
|
||||||
@@ -1020,6 +1051,9 @@ class LanceDBConnection(DBConnection):
|
|||||||
"""
|
"""
|
||||||
if namespace_path is None:
|
if namespace_path is None:
|
||||||
namespace_path = []
|
namespace_path = []
|
||||||
|
if namespace_path:
|
||||||
|
self._namespace_conn().drop_table(name, namespace_path=namespace_path)
|
||||||
|
return
|
||||||
LOOP.run(
|
LOOP.run(
|
||||||
self._conn.drop_table(
|
self._conn.drop_table(
|
||||||
name, namespace_path=namespace_path, ignore_missing=ignore_missing
|
name, namespace_path=namespace_path, ignore_missing=ignore_missing
|
||||||
@@ -1071,14 +1105,17 @@ class LanceDBConnection(DBConnection):
|
|||||||
"""Get the equivalent namespace client for this connection.
|
"""Get the equivalent namespace client for this connection.
|
||||||
|
|
||||||
Returns a DirectoryNamespace pointing to the same root with the
|
Returns a DirectoryNamespace pointing to the same root with the
|
||||||
same storage options.
|
same storage options. The result is cached for the lifetime of
|
||||||
|
this connection.
|
||||||
|
|
||||||
Returns
|
Returns
|
||||||
-------
|
-------
|
||||||
LanceNamespace
|
LanceNamespace
|
||||||
The namespace client for this connection.
|
The namespace client for this connection.
|
||||||
"""
|
"""
|
||||||
return LOOP.run(self._conn.namespace_client())
|
if self._cached_namespace_client is None:
|
||||||
|
self._cached_namespace_client = LOOP.run(self._conn.namespace_client())
|
||||||
|
return self._cached_namespace_client
|
||||||
|
|
||||||
@deprecation.deprecated(
|
@deprecation.deprecated(
|
||||||
deprecated_in="0.15.1",
|
deprecated_in="0.15.1",
|
||||||
@@ -1353,6 +1390,7 @@ class AsyncConnection(object):
|
|||||||
namespace_path: Optional[List[str]] = None,
|
namespace_path: Optional[List[str]] = None,
|
||||||
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
|
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
|
||||||
location: Optional[str] = None,
|
location: Optional[str] = None,
|
||||||
|
namespace_client: Optional[Any] = None,
|
||||||
) -> AsyncTable:
|
) -> AsyncTable:
|
||||||
"""Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
|
"""Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
|
||||||
|
|
||||||
@@ -1397,7 +1435,7 @@ class AsyncConnection(object):
|
|||||||
Additional options for the storage backend. Options already set on the
|
Additional options for the storage backend. Options already set on the
|
||||||
connection will be inherited by the table, but can be overridden here.
|
connection will be inherited by the table, but can be overridden here.
|
||||||
See available options at
|
See available options at
|
||||||
<https://lancedb.com/docs/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
|
|
||||||
To enable stable row IDs (row IDs remain stable after compaction,
|
To enable stable row IDs (row IDs remain stable after compaction,
|
||||||
update, delete, and merges), set `new_table_enable_stable_row_ids`
|
update, delete, and merges), set `new_table_enable_stable_row_ids`
|
||||||
@@ -1550,6 +1588,7 @@ class AsyncConnection(object):
|
|||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
location=location,
|
location=location,
|
||||||
|
namespace_client=namespace_client,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
data = data_to_reader(data, schema)
|
data = data_to_reader(data, schema)
|
||||||
@@ -1560,6 +1599,7 @@ class AsyncConnection(object):
|
|||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
location=location,
|
location=location,
|
||||||
|
namespace_client=namespace_client,
|
||||||
)
|
)
|
||||||
|
|
||||||
return AsyncTable(new_table)
|
return AsyncTable(new_table)
|
||||||
@@ -1588,7 +1628,7 @@ class AsyncConnection(object):
|
|||||||
Additional options for the storage backend. Options already set on the
|
Additional options for the storage backend. Options already set on the
|
||||||
connection will be inherited by the table, but can be overridden here.
|
connection will be inherited by the table, but can be overridden here.
|
||||||
See available options at
|
See available options at
|
||||||
<https://lancedb.com/docs/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
index_cache_size: int, default 256
|
index_cache_size: int, default 256
|
||||||
**Deprecated**: Use session-level cache configuration instead.
|
**Deprecated**: Use session-level cache configuration instead.
|
||||||
Create a Session with custom cache sizes and pass it to lancedb.connect().
|
Create a Session with custom cache sizes and pass it to lancedb.connect().
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ through a namespace abstraction.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import sys
|
import sys
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
|
||||||
|
|
||||||
@@ -25,7 +24,24 @@ if TYPE_CHECKING:
|
|||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
import pyarrow as pa
|
import pyarrow as pa
|
||||||
|
|
||||||
from lancedb.db import DBConnection, LanceDBConnection
|
from lance_namespace_urllib3_client.models.json_arrow_data_type import JsonArrowDataType
|
||||||
|
from lance_namespace_urllib3_client.models.json_arrow_field import JsonArrowField
|
||||||
|
from lance_namespace_urllib3_client.models.json_arrow_schema import JsonArrowSchema
|
||||||
|
from lance_namespace_urllib3_client.models.query_table_request import QueryTableRequest
|
||||||
|
from lance_namespace_urllib3_client.models.query_table_request_columns import (
|
||||||
|
QueryTableRequestColumns,
|
||||||
|
)
|
||||||
|
from lance_namespace_urllib3_client.models.query_table_request_full_text_query import (
|
||||||
|
QueryTableRequestFullTextQuery,
|
||||||
|
)
|
||||||
|
from lance_namespace_urllib3_client.models.query_table_request_vector import (
|
||||||
|
QueryTableRequestVector,
|
||||||
|
)
|
||||||
|
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
|
||||||
|
from lance_namespace.errors import TableNotFoundError
|
||||||
|
from lancedb._lancedb import connect_namespace_client as _connect_namespace_client
|
||||||
|
from lancedb.background_loop import LOOP
|
||||||
|
from lancedb.db import AsyncConnection, DBConnection
|
||||||
from lancedb.namespace_utils import (
|
from lancedb.namespace_utils import (
|
||||||
_normalize_create_namespace_mode,
|
_normalize_create_namespace_mode,
|
||||||
_normalize_drop_namespace_mode,
|
_normalize_drop_namespace_mode,
|
||||||
@@ -40,14 +56,11 @@ from lance_namespace import (
|
|||||||
ListNamespacesResponse,
|
ListNamespacesResponse,
|
||||||
ListTablesResponse,
|
ListTablesResponse,
|
||||||
ListTablesRequest,
|
ListTablesRequest,
|
||||||
DescribeTableRequest,
|
|
||||||
DescribeNamespaceRequest,
|
DescribeNamespaceRequest,
|
||||||
DropTableRequest,
|
DropTableRequest,
|
||||||
ListNamespacesRequest,
|
ListNamespacesRequest,
|
||||||
CreateNamespaceRequest,
|
CreateNamespaceRequest,
|
||||||
DropNamespaceRequest,
|
DropNamespaceRequest,
|
||||||
DeclareTableRequest,
|
|
||||||
CreateTableRequest,
|
|
||||||
)
|
)
|
||||||
from lancedb.table import AsyncTable, LanceTable, Table
|
from lancedb.table import AsyncTable, LanceTable, Table
|
||||||
from lancedb.util import validate_table_name
|
from lancedb.util import validate_table_name
|
||||||
@@ -56,21 +69,6 @@ from lancedb.pydantic import LanceModel
|
|||||||
from lancedb.embeddings import EmbeddingFunctionConfig
|
from lancedb.embeddings import EmbeddingFunctionConfig
|
||||||
from ._lancedb import Session
|
from ._lancedb import Session
|
||||||
|
|
||||||
from lance_namespace_urllib3_client.models.json_arrow_schema import JsonArrowSchema
|
|
||||||
from lance_namespace_urllib3_client.models.json_arrow_field import JsonArrowField
|
|
||||||
from lance_namespace_urllib3_client.models.json_arrow_data_type import JsonArrowDataType
|
|
||||||
from lance_namespace_urllib3_client.models.query_table_request import QueryTableRequest
|
|
||||||
from lance_namespace_urllib3_client.models.query_table_request_vector import (
|
|
||||||
QueryTableRequestVector,
|
|
||||||
)
|
|
||||||
from lance_namespace_urllib3_client.models.query_table_request_columns import (
|
|
||||||
QueryTableRequestColumns,
|
|
||||||
)
|
|
||||||
from lance_namespace_urllib3_client.models.query_table_request_full_text_query import (
|
|
||||||
QueryTableRequestFullTextQuery,
|
|
||||||
)
|
|
||||||
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
|
|
||||||
|
|
||||||
|
|
||||||
def _query_to_namespace_request(
|
def _query_to_namespace_request(
|
||||||
table_id: List[str],
|
table_id: List[str],
|
||||||
@@ -381,6 +379,8 @@ class LanceNamespaceDBConnection(DBConnection):
|
|||||||
storage_options: Optional[Dict[str, str]] = None,
|
storage_options: Optional[Dict[str, str]] = None,
|
||||||
session: Optional[Session] = None,
|
session: Optional[Session] = None,
|
||||||
namespace_client_pushdown_operations: Optional[List[str]] = None,
|
namespace_client_pushdown_operations: Optional[List[str]] = None,
|
||||||
|
namespace_client_impl: Optional[str] = None,
|
||||||
|
namespace_client_properties: Optional[Dict[str, str]] = None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Initialize a namespace-based LanceDB connection.
|
Initialize a namespace-based LanceDB connection.
|
||||||
@@ -406,12 +406,60 @@ class LanceNamespaceDBConnection(DBConnection):
|
|||||||
namespace.create_table() instead of using declare_table + local write.
|
namespace.create_table() instead of using declare_table + local write.
|
||||||
|
|
||||||
Default is None (no pushdown, all operations run locally).
|
Default is None (no pushdown, all operations run locally).
|
||||||
|
namespace_client_impl : Optional[str]
|
||||||
|
The namespace implementation name used to create this connection.
|
||||||
|
Stored for serialization purposes.
|
||||||
|
namespace_client_properties : Optional[Dict[str, str]]
|
||||||
|
The namespace properties used to create this connection.
|
||||||
|
Stored for serialization purposes.
|
||||||
"""
|
"""
|
||||||
self._namespace_client = namespace_client
|
self._namespace_client = namespace_client
|
||||||
self.read_consistency_interval = read_consistency_interval
|
self.read_consistency_interval = read_consistency_interval
|
||||||
self.storage_options = storage_options or {}
|
self.storage_options = storage_options or {}
|
||||||
self.session = session
|
self.session = session
|
||||||
self._pushdown_operations = set(namespace_client_pushdown_operations or [])
|
self._namespace_client_pushdown_operations = set(
|
||||||
|
namespace_client_pushdown_operations or []
|
||||||
|
)
|
||||||
|
self._namespace_client_impl = namespace_client_impl
|
||||||
|
self._namespace_client_properties = namespace_client_properties
|
||||||
|
self._inner = AsyncConnection(
|
||||||
|
_connect_namespace_client(
|
||||||
|
namespace_client,
|
||||||
|
read_consistency_interval=(
|
||||||
|
read_consistency_interval.total_seconds()
|
||||||
|
if read_consistency_interval is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
storage_options=self.storage_options or None,
|
||||||
|
session=session,
|
||||||
|
namespace_client_pushdown_operations=(
|
||||||
|
list(self._namespace_client_pushdown_operations)
|
||||||
|
),
|
||||||
|
namespace_client_impl=namespace_client_impl,
|
||||||
|
namespace_client_properties=namespace_client_properties,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
@override
|
||||||
|
def serialize(self) -> str:
|
||||||
|
import json
|
||||||
|
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"connection_type": "namespace",
|
||||||
|
"namespace_client_impl": self._namespace_client_impl,
|
||||||
|
"namespace_client_properties": self._namespace_client_properties,
|
||||||
|
"namespace_client_pushdown_operations": sorted(
|
||||||
|
self._namespace_client_pushdown_operations
|
||||||
|
),
|
||||||
|
"storage_options": self.storage_options or None,
|
||||||
|
"read_consistency_interval_seconds": (
|
||||||
|
self.read_consistency_interval.total_seconds()
|
||||||
|
if self.read_consistency_interval
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
def table_names(
|
def table_names(
|
||||||
@@ -464,13 +512,10 @@ class LanceNamespaceDBConnection(DBConnection):
|
|||||||
if mode.lower() not in ["create", "overwrite"]:
|
if mode.lower() not in ["create", "overwrite"]:
|
||||||
raise ValueError("mode must be either 'create' or 'overwrite'")
|
raise ValueError("mode must be either 'create' or 'overwrite'")
|
||||||
validate_table_name(name)
|
validate_table_name(name)
|
||||||
|
async_table = LOOP.run(
|
||||||
table_id = namespace_path + [name]
|
self._inner.create_table(
|
||||||
|
name,
|
||||||
if "CreateTable" in self._pushdown_operations:
|
data,
|
||||||
return self._create_table_server_side(
|
|
||||||
name=name,
|
|
||||||
data=data,
|
|
||||||
schema=schema,
|
schema=schema,
|
||||||
mode=mode,
|
mode=mode,
|
||||||
exist_ok=exist_ok,
|
exist_ok=exist_ok,
|
||||||
@@ -480,127 +525,15 @@ class LanceNamespaceDBConnection(DBConnection):
|
|||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Local create path: declare_table + local write
|
|
||||||
# Step 1: Get the table location and storage options from namespace
|
|
||||||
# In overwrite mode, if table exists, use describe_table to get
|
|
||||||
# existing location. Otherwise, call create_empty_table to reserve
|
|
||||||
# a new location
|
|
||||||
location = None
|
|
||||||
namespace_storage_options = None
|
|
||||||
if mode.lower() == "overwrite":
|
|
||||||
# Try to describe the table first to see if it exists
|
|
||||||
try:
|
|
||||||
describe_request = DescribeTableRequest(id=table_id)
|
|
||||||
describe_response = self._namespace_client.describe_table(
|
|
||||||
describe_request
|
|
||||||
)
|
|
||||||
location = describe_response.location
|
|
||||||
namespace_storage_options = describe_response.storage_options
|
|
||||||
except Exception:
|
|
||||||
# Table doesn't exist, will create a new one below
|
|
||||||
pass
|
|
||||||
|
|
||||||
if location is None:
|
|
||||||
# Table doesn't exist or mode is "create", reserve a new location
|
|
||||||
declare_request = DeclareTableRequest(
|
|
||||||
id=table_id,
|
|
||||||
location=None,
|
|
||||||
properties=self.storage_options if self.storage_options else None,
|
|
||||||
)
|
|
||||||
declare_response = self._namespace_client.declare_table(declare_request)
|
|
||||||
|
|
||||||
if not declare_response.location:
|
|
||||||
raise ValueError(
|
|
||||||
"Table location is missing from declare_table response"
|
|
||||||
)
|
|
||||||
|
|
||||||
location = declare_response.location
|
|
||||||
namespace_storage_options = declare_response.storage_options
|
|
||||||
|
|
||||||
# Merge storage options: self.storage_options < user options < namespace options
|
|
||||||
merged_storage_options = dict(self.storage_options)
|
|
||||||
if storage_options:
|
|
||||||
merged_storage_options.update(storage_options)
|
|
||||||
if namespace_storage_options:
|
|
||||||
merged_storage_options.update(namespace_storage_options)
|
|
||||||
|
|
||||||
# Step 2: Create table using LanceTable.create with the location
|
|
||||||
# We need a temporary connection for the LanceTable.create method
|
|
||||||
temp_conn = LanceDBConnection(
|
|
||||||
location, # Use the actual location as the connection URI
|
|
||||||
read_consistency_interval=self.read_consistency_interval,
|
|
||||||
storage_options=merged_storage_options,
|
|
||||||
session=self.session,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Note: storage_options_provider is auto-created in Rust from namespace_client
|
return LanceTable(
|
||||||
tbl = LanceTable.create(
|
self,
|
||||||
temp_conn,
|
|
||||||
name,
|
name,
|
||||||
data,
|
|
||||||
schema,
|
|
||||||
mode=mode,
|
|
||||||
exist_ok=exist_ok,
|
|
||||||
on_bad_vectors=on_bad_vectors,
|
|
||||||
fill_value=fill_value,
|
|
||||||
embedding_functions=embedding_functions,
|
|
||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=merged_storage_options,
|
|
||||||
location=location,
|
|
||||||
namespace_client=self._namespace_client,
|
namespace_client=self._namespace_client,
|
||||||
pushdown_operations=self._pushdown_operations,
|
pushdown_operations=self._namespace_client_pushdown_operations,
|
||||||
)
|
_async=async_table,
|
||||||
|
|
||||||
return tbl
|
|
||||||
|
|
||||||
def _create_table_server_side(
|
|
||||||
self,
|
|
||||||
name: str,
|
|
||||||
data: Optional[DATA],
|
|
||||||
schema: Optional[Union[pa.Schema, LanceModel]],
|
|
||||||
mode: str,
|
|
||||||
exist_ok: bool,
|
|
||||||
on_bad_vectors: str,
|
|
||||||
fill_value: float,
|
|
||||||
embedding_functions: Optional[List[EmbeddingFunctionConfig]],
|
|
||||||
namespace_path: Optional[List[str]],
|
|
||||||
storage_options: Optional[Dict[str, str]],
|
|
||||||
) -> Table:
|
|
||||||
"""Create a table using server-side namespace.create_table()."""
|
|
||||||
if namespace_path is None:
|
|
||||||
namespace_path = []
|
|
||||||
table_id = namespace_path + [name]
|
|
||||||
|
|
||||||
arrow_ipc_bytes = _data_to_arrow_ipc(
|
|
||||||
data=data,
|
|
||||||
schema=schema,
|
|
||||||
embedding_functions=embedding_functions,
|
|
||||||
on_bad_vectors=on_bad_vectors,
|
|
||||||
fill_value=fill_value,
|
|
||||||
)
|
|
||||||
|
|
||||||
request = CreateTableRequest(
|
|
||||||
id=table_id,
|
|
||||||
mode=_normalize_create_table_mode(mode),
|
|
||||||
properties=self.storage_options if self.storage_options else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
self._namespace_client.create_table(request, arrow_ipc_bytes)
|
|
||||||
except Exception as e:
|
|
||||||
if exist_ok and "already exists" in str(e).lower():
|
|
||||||
return self.open_table(
|
|
||||||
name,
|
|
||||||
namespace_path=namespace_path,
|
|
||||||
storage_options=storage_options,
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
|
|
||||||
return self.open_table(
|
|
||||||
name,
|
|
||||||
namespace_path=namespace_path,
|
|
||||||
storage_options=storage_options,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -614,30 +547,28 @@ class LanceNamespaceDBConnection(DBConnection):
|
|||||||
) -> Table:
|
) -> Table:
|
||||||
if namespace_path is None:
|
if namespace_path is None:
|
||||||
namespace_path = []
|
namespace_path = []
|
||||||
table_id = namespace_path + [name]
|
try:
|
||||||
request = DescribeTableRequest(id=table_id)
|
async_table = LOOP.run(
|
||||||
response = self._namespace_client.describe_table(request)
|
self._inner.open_table(
|
||||||
|
name,
|
||||||
|
namespace_path=namespace_path,
|
||||||
|
storage_options=storage_options,
|
||||||
|
index_cache_size=index_cache_size,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except RuntimeError as e:
|
||||||
|
if "Table not found" in str(e):
|
||||||
|
table_id = namespace_path + [name]
|
||||||
|
raise TableNotFoundError(f"Table not found: {'$'.join(table_id)}")
|
||||||
|
raise
|
||||||
|
|
||||||
# Merge storage options: self.storage_options < user options < namespace options
|
return LanceTable(
|
||||||
merged_storage_options = dict(self.storage_options)
|
self,
|
||||||
if storage_options:
|
|
||||||
merged_storage_options.update(storage_options)
|
|
||||||
if response.storage_options:
|
|
||||||
merged_storage_options.update(response.storage_options)
|
|
||||||
|
|
||||||
# Pass managed_versioning to avoid redundant describe_table call in Rust.
|
|
||||||
# Convert None to False since we already have the answer from describe_table.
|
|
||||||
managed_versioning = response.managed_versioning is True
|
|
||||||
|
|
||||||
# Note: storage_options_provider is auto-created in Rust from namespace_client
|
|
||||||
return self._lance_table_from_uri(
|
|
||||||
name,
|
name,
|
||||||
response.location,
|
|
||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=merged_storage_options,
|
|
||||||
index_cache_size=index_cache_size,
|
|
||||||
namespace_client=self._namespace_client,
|
namespace_client=self._namespace_client,
|
||||||
managed_versioning=managed_versioning,
|
pushdown_operations=self._namespace_client_pushdown_operations,
|
||||||
|
_async=async_table,
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -861,33 +792,34 @@ class LanceNamespaceDBConnection(DBConnection):
|
|||||||
namespace_client: Optional[Any] = None,
|
namespace_client: Optional[Any] = None,
|
||||||
managed_versioning: Optional[bool] = None,
|
managed_versioning: Optional[bool] = None,
|
||||||
) -> LanceTable:
|
) -> LanceTable:
|
||||||
# Open a table directly from a URI using the location parameter
|
# Open a table directly from the namespace-resolved physical location.
|
||||||
# Note: storage_options should already be merged by the caller
|
#
|
||||||
# Note: storage_options_provider is auto-created in Rust from namespace_client
|
# Open the table through the Rust namespace-backed connection. The Rust
|
||||||
|
# layer keeps the logical namespace path and namespace client intact.
|
||||||
if namespace_path is None:
|
if namespace_path is None:
|
||||||
namespace_path = []
|
namespace_path = []
|
||||||
temp_conn = LanceDBConnection(
|
|
||||||
table_uri, # Use the table location as the connection URI
|
async_table = LOOP.run(
|
||||||
read_consistency_interval=self.read_consistency_interval,
|
self._inner.open_table(
|
||||||
storage_options=storage_options if storage_options is not None else {},
|
name,
|
||||||
session=self.session,
|
namespace_path=namespace_path,
|
||||||
|
storage_options=storage_options,
|
||||||
|
index_cache_size=index_cache_size,
|
||||||
|
location=None,
|
||||||
|
namespace_client=namespace_client,
|
||||||
|
managed_versioning=managed_versioning,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# Open the table using the temporary connection with the location parameter
|
return LanceTable(
|
||||||
# Pass namespace_client to enable managed versioning support and auto-create
|
self,
|
||||||
# storage options provider
|
|
||||||
# Pass managed_versioning to avoid redundant describe_table call
|
|
||||||
# Pass pushdown_operations if configured on this connection
|
|
||||||
return LanceTable.open(
|
|
||||||
temp_conn,
|
|
||||||
name,
|
name,
|
||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=storage_options,
|
|
||||||
index_cache_size=index_cache_size,
|
|
||||||
location=table_uri,
|
location=table_uri,
|
||||||
namespace_client=namespace_client,
|
namespace_client=namespace_client,
|
||||||
managed_versioning=managed_versioning,
|
managed_versioning=managed_versioning,
|
||||||
pushdown_operations=self._pushdown_operations,
|
pushdown_operations=self._namespace_client_pushdown_operations,
|
||||||
|
_async=async_table,
|
||||||
)
|
)
|
||||||
|
|
||||||
@override
|
@override
|
||||||
@@ -951,7 +883,26 @@ class AsyncLanceNamespaceDBConnection:
|
|||||||
self.read_consistency_interval = read_consistency_interval
|
self.read_consistency_interval = read_consistency_interval
|
||||||
self.storage_options = storage_options or {}
|
self.storage_options = storage_options or {}
|
||||||
self.session = session
|
self.session = session
|
||||||
self._pushdown_operations = set(namespace_client_pushdown_operations or [])
|
self._namespace_client_pushdown_operations = set(
|
||||||
|
namespace_client_pushdown_operations or []
|
||||||
|
)
|
||||||
|
self._inner = AsyncConnection(
|
||||||
|
_connect_namespace_client(
|
||||||
|
namespace_client,
|
||||||
|
read_consistency_interval=(
|
||||||
|
read_consistency_interval.total_seconds()
|
||||||
|
if read_consistency_interval is not None
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
storage_options=self.storage_options or None,
|
||||||
|
session=session,
|
||||||
|
namespace_client_pushdown_operations=(
|
||||||
|
list(self._namespace_client_pushdown_operations)
|
||||||
|
),
|
||||||
|
namespace_client_impl=None,
|
||||||
|
namespace_client_properties=None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
async def table_names(
|
async def table_names(
|
||||||
self,
|
self,
|
||||||
@@ -1003,145 +954,16 @@ class AsyncLanceNamespaceDBConnection:
|
|||||||
if mode.lower() not in ["create", "overwrite"]:
|
if mode.lower() not in ["create", "overwrite"]:
|
||||||
raise ValueError("mode must be either 'create' or 'overwrite'")
|
raise ValueError("mode must be either 'create' or 'overwrite'")
|
||||||
validate_table_name(name)
|
validate_table_name(name)
|
||||||
|
return await self._inner.create_table(
|
||||||
table_id = namespace_path + [name]
|
|
||||||
|
|
||||||
if "CreateTable" in self._pushdown_operations:
|
|
||||||
return await self._create_table_server_side(
|
|
||||||
name=name,
|
|
||||||
data=data,
|
|
||||||
schema=schema,
|
|
||||||
mode=mode,
|
|
||||||
exist_ok=exist_ok,
|
|
||||||
on_bad_vectors=on_bad_vectors,
|
|
||||||
fill_value=fill_value,
|
|
||||||
embedding_functions=embedding_functions,
|
|
||||||
namespace_path=namespace_path,
|
|
||||||
storage_options=storage_options,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Local create path: declare_table + local write
|
|
||||||
# Step 1: Get the table location and storage options from namespace
|
|
||||||
location = None
|
|
||||||
namespace_storage_options = None
|
|
||||||
if mode.lower() == "overwrite":
|
|
||||||
# Try to describe the table first to see if it exists
|
|
||||||
try:
|
|
||||||
describe_request = DescribeTableRequest(id=table_id)
|
|
||||||
describe_response = self._namespace_client.describe_table(
|
|
||||||
describe_request
|
|
||||||
)
|
|
||||||
location = describe_response.location
|
|
||||||
namespace_storage_options = describe_response.storage_options
|
|
||||||
except Exception:
|
|
||||||
# Table doesn't exist, will create a new one below
|
|
||||||
pass
|
|
||||||
|
|
||||||
if location is None:
|
|
||||||
# Table doesn't exist or mode is "create", reserve a new location
|
|
||||||
declare_request = DeclareTableRequest(
|
|
||||||
id=table_id,
|
|
||||||
location=None,
|
|
||||||
properties=self.storage_options if self.storage_options else None,
|
|
||||||
)
|
|
||||||
declare_response = self._namespace_client.declare_table(declare_request)
|
|
||||||
|
|
||||||
if not declare_response.location:
|
|
||||||
raise ValueError(
|
|
||||||
"Table location is missing from declare_table response"
|
|
||||||
)
|
|
||||||
|
|
||||||
location = declare_response.location
|
|
||||||
namespace_storage_options = declare_response.storage_options
|
|
||||||
|
|
||||||
# Merge storage options: self.storage_options < user options < namespace options
|
|
||||||
merged_storage_options = dict(self.storage_options)
|
|
||||||
if storage_options:
|
|
||||||
merged_storage_options.update(storage_options)
|
|
||||||
if namespace_storage_options:
|
|
||||||
merged_storage_options.update(namespace_storage_options)
|
|
||||||
|
|
||||||
# Step 2: Create table using LanceTable.create with the location
|
|
||||||
# Run the sync operation in a thread
|
|
||||||
def _create_table():
|
|
||||||
temp_conn = LanceDBConnection(
|
|
||||||
location,
|
|
||||||
read_consistency_interval=self.read_consistency_interval,
|
|
||||||
storage_options=merged_storage_options,
|
|
||||||
session=self.session,
|
|
||||||
)
|
|
||||||
|
|
||||||
# storage_options_provider is auto-created in Rust from namespace_client
|
|
||||||
return LanceTable.create(
|
|
||||||
temp_conn,
|
|
||||||
name,
|
|
||||||
data,
|
|
||||||
schema,
|
|
||||||
mode=mode,
|
|
||||||
exist_ok=exist_ok,
|
|
||||||
on_bad_vectors=on_bad_vectors,
|
|
||||||
fill_value=fill_value,
|
|
||||||
embedding_functions=embedding_functions,
|
|
||||||
namespace_path=namespace_path,
|
|
||||||
storage_options=merged_storage_options,
|
|
||||||
location=location,
|
|
||||||
namespace_client=self._namespace_client,
|
|
||||||
pushdown_operations=self._pushdown_operations,
|
|
||||||
)
|
|
||||||
|
|
||||||
lance_table = await asyncio.to_thread(_create_table)
|
|
||||||
# Get the underlying async table from LanceTable
|
|
||||||
return lance_table._table
|
|
||||||
|
|
||||||
async def _create_table_server_side(
|
|
||||||
self,
|
|
||||||
name: str,
|
|
||||||
data: Optional[DATA],
|
|
||||||
schema: Optional[Union[pa.Schema, LanceModel]],
|
|
||||||
mode: str,
|
|
||||||
exist_ok: bool,
|
|
||||||
on_bad_vectors: str,
|
|
||||||
fill_value: float,
|
|
||||||
embedding_functions: Optional[List[EmbeddingFunctionConfig]],
|
|
||||||
namespace_path: Optional[List[str]],
|
|
||||||
storage_options: Optional[Dict[str, str]],
|
|
||||||
) -> AsyncTable:
|
|
||||||
"""Create a table using server-side namespace.create_table()."""
|
|
||||||
if namespace_path is None:
|
|
||||||
namespace_path = []
|
|
||||||
table_id = namespace_path + [name]
|
|
||||||
|
|
||||||
def _prepare_and_create():
|
|
||||||
arrow_ipc_bytes = _data_to_arrow_ipc(
|
|
||||||
data=data,
|
|
||||||
schema=schema,
|
|
||||||
embedding_functions=embedding_functions,
|
|
||||||
on_bad_vectors=on_bad_vectors,
|
|
||||||
fill_value=fill_value,
|
|
||||||
)
|
|
||||||
|
|
||||||
request = CreateTableRequest(
|
|
||||||
id=table_id,
|
|
||||||
mode=_normalize_create_table_mode(mode),
|
|
||||||
properties=self.storage_options if self.storage_options else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
self._namespace_client.create_table(request, arrow_ipc_bytes)
|
|
||||||
|
|
||||||
try:
|
|
||||||
await asyncio.to_thread(_prepare_and_create)
|
|
||||||
except Exception as e:
|
|
||||||
if exist_ok and "already exists" in str(e).lower():
|
|
||||||
return await self.open_table(
|
|
||||||
name,
|
|
||||||
namespace_path=namespace_path,
|
|
||||||
storage_options=storage_options,
|
|
||||||
)
|
|
||||||
raise
|
|
||||||
|
|
||||||
return await self.open_table(
|
|
||||||
name,
|
name,
|
||||||
|
data,
|
||||||
|
schema=schema,
|
||||||
|
mode=mode,
|
||||||
|
exist_ok=exist_ok,
|
||||||
|
on_bad_vectors=on_bad_vectors,
|
||||||
|
fill_value=fill_value,
|
||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
|
embedding_functions=embedding_functions,
|
||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1156,45 +978,18 @@ class AsyncLanceNamespaceDBConnection:
|
|||||||
"""Open an existing table from the namespace."""
|
"""Open an existing table from the namespace."""
|
||||||
if namespace_path is None:
|
if namespace_path is None:
|
||||||
namespace_path = []
|
namespace_path = []
|
||||||
table_id = namespace_path + [name]
|
try:
|
||||||
request = DescribeTableRequest(id=table_id)
|
return await self._inner.open_table(
|
||||||
response = self._namespace_client.describe_table(request)
|
|
||||||
|
|
||||||
# Merge storage options: self.storage_options < user options < namespace options
|
|
||||||
merged_storage_options = dict(self.storage_options)
|
|
||||||
if storage_options:
|
|
||||||
merged_storage_options.update(storage_options)
|
|
||||||
if response.storage_options:
|
|
||||||
merged_storage_options.update(response.storage_options)
|
|
||||||
|
|
||||||
# Capture managed_versioning from describe response.
|
|
||||||
# Convert None to False since we already have the answer from describe_table.
|
|
||||||
managed_versioning = response.managed_versioning is True
|
|
||||||
|
|
||||||
# Open table in a thread
|
|
||||||
# Note: storage_options_provider is auto-created in Rust from namespace_client
|
|
||||||
def _open_table():
|
|
||||||
temp_conn = LanceDBConnection(
|
|
||||||
response.location,
|
|
||||||
read_consistency_interval=self.read_consistency_interval,
|
|
||||||
storage_options=merged_storage_options,
|
|
||||||
session=self.session,
|
|
||||||
)
|
|
||||||
|
|
||||||
return LanceTable.open(
|
|
||||||
temp_conn,
|
|
||||||
name,
|
name,
|
||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=merged_storage_options,
|
storage_options=storage_options,
|
||||||
index_cache_size=index_cache_size,
|
index_cache_size=index_cache_size,
|
||||||
location=response.location,
|
|
||||||
namespace_client=self._namespace_client,
|
|
||||||
managed_versioning=managed_versioning,
|
|
||||||
pushdown_operations=self._pushdown_operations,
|
|
||||||
)
|
)
|
||||||
|
except RuntimeError as e:
|
||||||
lance_table = await asyncio.to_thread(_open_table)
|
if "Table not found" in str(e):
|
||||||
return lance_table._table
|
table_id = namespace_path + [name]
|
||||||
|
raise TableNotFoundError(f"Table not found: {'$'.join(table_id)}")
|
||||||
|
raise
|
||||||
|
|
||||||
async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
|
async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
|
||||||
"""Drop a table from the namespace."""
|
"""Drop a table from the namespace."""
|
||||||
@@ -1472,6 +1267,8 @@ def connect_namespace(
|
|||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
session=session,
|
session=session,
|
||||||
namespace_client_pushdown_operations=namespace_client_pushdown_operations,
|
namespace_client_pushdown_operations=namespace_client_pushdown_operations,
|
||||||
|
namespace_client_impl=namespace_client_impl,
|
||||||
|
namespace_client_properties=namespace_client_properties,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -191,7 +191,7 @@ def _into_pyarrow_reader(
|
|||||||
f"Unknown data type {type(data)}. "
|
f"Unknown data type {type(data)}. "
|
||||||
"Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
|
"Supported types: list of dicts, pandas DataFrame, polars DataFrame, "
|
||||||
"pyarrow Table/RecordBatch, or Pydantic models. "
|
"pyarrow Table/RecordBatch, or Pydantic models. "
|
||||||
"See https://lancedb.com/docs/tables/ for examples."
|
"See https://docs.lancedb.com/tables/ for examples."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -2929,6 +2929,7 @@ class LanceTable(Table):
|
|||||||
namespace_path=namespace_path,
|
namespace_path=namespace_path,
|
||||||
storage_options=storage_options,
|
storage_options=storage_options,
|
||||||
location=location,
|
location=location,
|
||||||
|
namespace_client=namespace_client,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return self
|
return self
|
||||||
|
|||||||
@@ -897,42 +897,22 @@ def test_bypass_vector_index_sync(tmp_db: lancedb.DBConnection):
|
|||||||
|
|
||||||
|
|
||||||
def test_local_namespace_operations(tmp_path):
|
def test_local_namespace_operations(tmp_path):
|
||||||
"""Test that local mode namespace operations behave as expected."""
|
"""Test that local mode namespace operations work via directory namespace."""
|
||||||
# Create a local database connection
|
|
||||||
db = lancedb.connect(tmp_path)
|
db = lancedb.connect(tmp_path)
|
||||||
|
|
||||||
# Test list_namespaces returns empty list for root namespace
|
# Root namespace starts empty
|
||||||
namespaces = db.list_namespaces().namespaces
|
assert db.list_namespaces().namespaces == []
|
||||||
assert namespaces == []
|
|
||||||
|
|
||||||
# Test list_namespaces with non-empty namespace raises NotImplementedError
|
# Create and list child namespace
|
||||||
with pytest.raises(
|
db.create_namespace(["child"])
|
||||||
NotImplementedError,
|
assert "child" in db.list_namespaces().namespaces
|
||||||
match="Namespace operations are not supported for listing database",
|
|
||||||
):
|
|
||||||
db.list_namespaces(namespace_path=["test"])
|
|
||||||
|
|
||||||
|
# List namespaces under child
|
||||||
|
assert db.list_namespaces(namespace_path=["child"]).namespaces == []
|
||||||
|
|
||||||
def test_local_create_namespace_not_supported(tmp_path):
|
# Drop namespace
|
||||||
"""Test that create_namespace is not supported in local mode."""
|
db.drop_namespace(["child"])
|
||||||
db = lancedb.connect(tmp_path)
|
assert db.list_namespaces().namespaces == []
|
||||||
|
|
||||||
with pytest.raises(
|
|
||||||
NotImplementedError,
|
|
||||||
match="Namespace operations are not supported for listing database",
|
|
||||||
):
|
|
||||||
db.create_namespace(["test_namespace"])
|
|
||||||
|
|
||||||
|
|
||||||
def test_local_drop_namespace_not_supported(tmp_path):
|
|
||||||
"""Test that drop_namespace is not supported in local mode."""
|
|
||||||
db = lancedb.connect(tmp_path)
|
|
||||||
|
|
||||||
with pytest.raises(
|
|
||||||
NotImplementedError,
|
|
||||||
match="Namespace operations are not supported for listing database",
|
|
||||||
):
|
|
||||||
db.drop_namespace(["test_namespace"])
|
|
||||||
|
|
||||||
|
|
||||||
def test_clone_table_latest_version(tmp_path):
|
def test_clone_table_latest_version(tmp_path):
|
||||||
|
|||||||
@@ -681,7 +681,7 @@ class TestPushdownOperations:
|
|||||||
{"root": self.temp_dir},
|
{"root": self.temp_dir},
|
||||||
namespace_client_pushdown_operations=["QueryTable"],
|
namespace_client_pushdown_operations=["QueryTable"],
|
||||||
)
|
)
|
||||||
assert "QueryTable" in db._pushdown_operations
|
assert "QueryTable" in db._namespace_client_pushdown_operations
|
||||||
|
|
||||||
def test_create_table_pushdown_stored(self):
|
def test_create_table_pushdown_stored(self):
|
||||||
"""Test that CreateTable pushdown is stored on sync connection."""
|
"""Test that CreateTable pushdown is stored on sync connection."""
|
||||||
@@ -690,7 +690,7 @@ class TestPushdownOperations:
|
|||||||
{"root": self.temp_dir},
|
{"root": self.temp_dir},
|
||||||
namespace_client_pushdown_operations=["CreateTable"],
|
namespace_client_pushdown_operations=["CreateTable"],
|
||||||
)
|
)
|
||||||
assert "CreateTable" in db._pushdown_operations
|
assert "CreateTable" in db._namespace_client_pushdown_operations
|
||||||
|
|
||||||
def test_both_pushdowns_stored(self):
|
def test_both_pushdowns_stored(self):
|
||||||
"""Test that both pushdown operations can be set together."""
|
"""Test that both pushdown operations can be set together."""
|
||||||
@@ -699,13 +699,13 @@ class TestPushdownOperations:
|
|||||||
{"root": self.temp_dir},
|
{"root": self.temp_dir},
|
||||||
namespace_client_pushdown_operations=["QueryTable", "CreateTable"],
|
namespace_client_pushdown_operations=["QueryTable", "CreateTable"],
|
||||||
)
|
)
|
||||||
assert "QueryTable" in db._pushdown_operations
|
assert "QueryTable" in db._namespace_client_pushdown_operations
|
||||||
assert "CreateTable" in db._pushdown_operations
|
assert "CreateTable" in db._namespace_client_pushdown_operations
|
||||||
|
|
||||||
def test_pushdown_defaults_to_empty(self):
|
def test_pushdown_defaults_to_empty(self):
|
||||||
"""Test that pushdown operations default to empty."""
|
"""Test that pushdown operations default to empty."""
|
||||||
db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
|
db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
|
||||||
assert len(db._pushdown_operations) == 0
|
assert len(db._namespace_client_pushdown_operations) == 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -727,7 +727,7 @@ class TestAsyncPushdownOperations:
|
|||||||
{"root": self.temp_dir},
|
{"root": self.temp_dir},
|
||||||
namespace_client_pushdown_operations=["QueryTable"],
|
namespace_client_pushdown_operations=["QueryTable"],
|
||||||
)
|
)
|
||||||
assert "QueryTable" in db._pushdown_operations
|
assert "QueryTable" in db._namespace_client_pushdown_operations
|
||||||
|
|
||||||
async def test_async_create_table_pushdown_stored(self):
|
async def test_async_create_table_pushdown_stored(self):
|
||||||
"""Test that CreateTable pushdown is stored on async connection."""
|
"""Test that CreateTable pushdown is stored on async connection."""
|
||||||
@@ -736,9 +736,9 @@ class TestAsyncPushdownOperations:
|
|||||||
{"root": self.temp_dir},
|
{"root": self.temp_dir},
|
||||||
namespace_client_pushdown_operations=["CreateTable"],
|
namespace_client_pushdown_operations=["CreateTable"],
|
||||||
)
|
)
|
||||||
assert "CreateTable" in db._pushdown_operations
|
assert "CreateTable" in db._namespace_client_pushdown_operations
|
||||||
|
|
||||||
async def test_async_pushdown_defaults_to_empty(self):
|
async def test_async_pushdown_defaults_to_empty(self):
|
||||||
"""Test that pushdown operations default to empty on async connection."""
|
"""Test that pushdown operations default to empty on async connection."""
|
||||||
db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
|
db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
|
||||||
assert len(db._pushdown_operations) == 0
|
assert len(db._namespace_client_pushdown_operations) == 0
|
||||||
|
|||||||
@@ -18,6 +18,9 @@ Tests verify:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from typing import Dict, Optional
|
from typing import Dict, Optional
|
||||||
@@ -387,6 +390,66 @@ def test_namespace_open_table_with_provider(s3_bucket: str, use_custom: bool):
|
|||||||
assert get_describe_call_count(inner_ns_client) == describe_count_after_open
|
assert get_describe_call_count(inner_ns_client) == describe_count_after_open
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
sys.platform == "win32",
|
||||||
|
reason="TODO: fix schema-only namespace metrics test on Windows",
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
|
||||||
|
def test_namespace_create_schema_only_with_provider(use_custom: bool):
|
||||||
|
"""
|
||||||
|
Test creating a schema-only table through namespace with storage options provider.
|
||||||
|
|
||||||
|
Verifies:
|
||||||
|
- declare_table is called once to reserve the location
|
||||||
|
- describe_table is not needed during create in create mode
|
||||||
|
- the table can be reopened successfully afterward
|
||||||
|
- opening the table triggers exactly one describe_table call
|
||||||
|
"""
|
||||||
|
temp_dir = tempfile.mkdtemp()
|
||||||
|
try:
|
||||||
|
ns_client, inner_ns_client = create_tracking_namespace(
|
||||||
|
bucket_name=temp_dir,
|
||||||
|
storage_options={},
|
||||||
|
credential_expires_in_seconds=3600,
|
||||||
|
use_custom=use_custom,
|
||||||
|
)
|
||||||
|
|
||||||
|
db = LanceNamespaceDBConnection(ns_client)
|
||||||
|
|
||||||
|
namespace_name = f"test_ns_{uuid.uuid4().hex[:8]}"
|
||||||
|
db.create_namespace([namespace_name])
|
||||||
|
|
||||||
|
table_name = f"test_table_{uuid.uuid4().hex}"
|
||||||
|
namespace_path = [namespace_name]
|
||||||
|
schema = pa.schema(
|
||||||
|
[
|
||||||
|
pa.field("id", pa.int64()),
|
||||||
|
pa.field("vector", pa.list_(pa.float32(), 2)),
|
||||||
|
pa.field("text", pa.string()),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert get_declare_call_count(inner_ns_client) == 0
|
||||||
|
assert get_describe_call_count(inner_ns_client) == 0
|
||||||
|
|
||||||
|
table = db.create_table(
|
||||||
|
table_name, schema=schema, namespace_path=namespace_path
|
||||||
|
)
|
||||||
|
|
||||||
|
assert table.name == table_name
|
||||||
|
assert table.namespace == namespace_path
|
||||||
|
assert get_declare_call_count(inner_ns_client) == 1
|
||||||
|
assert get_describe_call_count(inner_ns_client) == 0
|
||||||
|
|
||||||
|
reopened_table = db.open_table(table_name, namespace_path=namespace_path)
|
||||||
|
|
||||||
|
assert reopened_table.schema == schema
|
||||||
|
assert get_declare_call_count(inner_ns_client) == 1
|
||||||
|
assert get_describe_call_count(inner_ns_client) == 1
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.s3_test
|
@pytest.mark.s3_test
|
||||||
@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
|
@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
|
||||||
def test_namespace_credential_refresh_on_read(s3_bucket: str, use_custom: bool):
|
def test_namespace_credential_refresh_on_read(s3_bucket: str, use_custom: bool):
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
|
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
from datetime import date, datetime, timedelta
|
from datetime import date, datetime, timedelta
|
||||||
from time import sleep
|
from time import sleep
|
||||||
from typing import List
|
from typing import List
|
||||||
@@ -2040,6 +2041,13 @@ def test_hybrid_search_metric_type(tmp_db: DBConnection):
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"consistency_interval", [None, timedelta(seconds=0), timedelta(seconds=0.1)]
|
"consistency_interval", [None, timedelta(seconds=0), timedelta(seconds=0.1)]
|
||||||
)
|
)
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
sys.platform == "win32",
|
||||||
|
reason=(
|
||||||
|
"TODO: directory namespace is not supported on Windows yet; "
|
||||||
|
"re-enable after that is fixed."
|
||||||
|
),
|
||||||
|
)
|
||||||
def test_consistency(tmp_path, consistency_interval):
|
def test_consistency(tmp_path, consistency_interval):
|
||||||
db = lancedb.connect(tmp_path)
|
db = lancedb.connect(tmp_path)
|
||||||
table = db.create_table("my_table", data=[{"id": 0}])
|
table = db.create_table("my_table", data=[{"id": 0}])
|
||||||
@@ -2060,7 +2068,6 @@ def test_consistency(tmp_path, consistency_interval):
|
|||||||
elif consistency_interval == timedelta(seconds=0):
|
elif consistency_interval == timedelta(seconds=0):
|
||||||
assert table2.version == table.version
|
assert table2.version == table.version
|
||||||
else:
|
else:
|
||||||
# (consistency_interval == timedelta(seconds=0.1)
|
|
||||||
assert table2.version == table.version - 1
|
assert table2.version == table.version - 1
|
||||||
sleep(0.1)
|
sleep(0.1)
|
||||||
assert table2.version == table.version
|
assert table2.version == table.version
|
||||||
|
|||||||
@@ -1,11 +1,17 @@
|
|||||||
// SPDX-License-Identifier: Apache-2.0
|
// SPDX-License-Identifier: Apache-2.0
|
||||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
use std::{collections::HashMap, sync::Arc, time::Duration};
|
use std::{
|
||||||
|
collections::{HashMap, HashSet},
|
||||||
|
sync::Arc,
|
||||||
|
time::Duration,
|
||||||
|
};
|
||||||
|
|
||||||
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
|
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
|
||||||
use lancedb::{
|
use lancedb::{
|
||||||
connection::Connection as LanceConnection,
|
connection::Connection as LanceConnection,
|
||||||
|
connection::NamespaceClientPushdownOperation,
|
||||||
|
database::namespace::LanceNamespaceDatabase,
|
||||||
database::{CreateTableMode, Database, ReadConsistency},
|
database::{CreateTableMode, Database, ReadConsistency},
|
||||||
};
|
};
|
||||||
use pyo3::{
|
use pyo3::{
|
||||||
@@ -39,6 +45,29 @@ impl Connection {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_namespace_client_pushdown_operations(
|
||||||
|
operations: Option<Vec<String>>,
|
||||||
|
) -> PyResult<HashSet<NamespaceClientPushdownOperation>> {
|
||||||
|
let mut parsed = HashSet::new();
|
||||||
|
for operation in operations.unwrap_or_default() {
|
||||||
|
match operation.as_str() {
|
||||||
|
"QueryTable" => {
|
||||||
|
parsed.insert(NamespaceClientPushdownOperation::QueryTable);
|
||||||
|
}
|
||||||
|
"CreateTable" => {
|
||||||
|
parsed.insert(NamespaceClientPushdownOperation::CreateTable);
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
return Err(PyValueError::new_err(format!(
|
||||||
|
"Invalid pushdown operation: {}",
|
||||||
|
operation
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(parsed)
|
||||||
|
}
|
||||||
|
|
||||||
impl Connection {
|
impl Connection {
|
||||||
fn parse_create_mode_str(mode: &str) -> PyResult<CreateTableMode> {
|
fn parse_create_mode_str(mode: &str) -> PyResult<CreateTableMode> {
|
||||||
match mode {
|
match mode {
|
||||||
@@ -538,6 +567,52 @@ pub fn connect(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[pyfunction]
|
||||||
|
#[pyo3(signature = (
|
||||||
|
namespace_client,
|
||||||
|
read_consistency_interval=None,
|
||||||
|
storage_options=None,
|
||||||
|
session=None,
|
||||||
|
namespace_client_pushdown_operations=None,
|
||||||
|
namespace_client_impl=None,
|
||||||
|
namespace_client_properties=None,
|
||||||
|
))]
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub fn connect_namespace_client(
|
||||||
|
py: Python<'_>,
|
||||||
|
namespace_client: Py<PyAny>,
|
||||||
|
read_consistency_interval: Option<f64>,
|
||||||
|
storage_options: Option<HashMap<String, String>>,
|
||||||
|
session: Option<crate::session::Session>,
|
||||||
|
namespace_client_pushdown_operations: Option<Vec<String>>,
|
||||||
|
namespace_client_impl: Option<String>,
|
||||||
|
namespace_client_properties: Option<HashMap<String, String>>,
|
||||||
|
) -> PyResult<Connection> {
|
||||||
|
let namespace_client = extract_namespace_arc(py, namespace_client)?;
|
||||||
|
let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
|
||||||
|
let namespace_client_pushdown_operations =
|
||||||
|
parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
|
||||||
|
let ns_impl = namespace_client_impl.unwrap_or_else(|| "python".to_string());
|
||||||
|
let ns_properties = namespace_client_properties.unwrap_or_default();
|
||||||
|
let storage_options = storage_options.unwrap_or_default();
|
||||||
|
let session = session.map(|s| s.inner.clone());
|
||||||
|
|
||||||
|
let database = LanceNamespaceDatabase::from_namespace_client(
|
||||||
|
namespace_client,
|
||||||
|
ns_impl,
|
||||||
|
ns_properties,
|
||||||
|
storage_options,
|
||||||
|
read_consistency_interval,
|
||||||
|
session,
|
||||||
|
namespace_client_pushdown_operations,
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Connection::new(LanceConnection::new(
|
||||||
|
Arc::new(database),
|
||||||
|
Arc::new(lancedb::embeddings::MemoryRegistry::new()),
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(FromPyObject)]
|
#[derive(FromPyObject)]
|
||||||
pub struct PyClientConfig {
|
pub struct PyClientConfig {
|
||||||
user_agent: String,
|
user_agent: String,
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||||
|
|
||||||
use arrow::RecordBatchStream;
|
use arrow::RecordBatchStream;
|
||||||
use connection::{Connection, connect};
|
use connection::{Connection, connect, connect_namespace_client};
|
||||||
use env_logger::Env;
|
use env_logger::Env;
|
||||||
use expr::{PyExpr, expr_col, expr_func, expr_lit};
|
use expr::{PyExpr, expr_col, expr_func, expr_lit};
|
||||||
use index::IndexConfig;
|
use index::IndexConfig;
|
||||||
@@ -58,6 +58,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
|
|||||||
m.add_class::<PyPermutationReader>()?;
|
m.add_class::<PyPermutationReader>()?;
|
||||||
m.add_class::<PyExpr>()?;
|
m.add_class::<PyExpr>()?;
|
||||||
m.add_function(wrap_pyfunction!(connect, m)?)?;
|
m.add_function(wrap_pyfunction!(connect, m)?)?;
|
||||||
|
m.add_function(wrap_pyfunction!(connect_namespace_client, m)?)?;
|
||||||
m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?;
|
m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?;
|
||||||
m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
|
m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
|
||||||
m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?;
|
m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?;
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "lancedb"
|
name = "lancedb"
|
||||||
version = "0.28.0-beta.2"
|
version = "0.28.0-beta.8"
|
||||||
edition.workspace = true
|
edition.workspace = true
|
||||||
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
description = "LanceDB: A serverless, low-latency vector database for AI applications"
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
|
|||||||
@@ -171,7 +171,7 @@ impl OpenTableBuilder {
|
|||||||
/// Options already set on the connection will be inherited by the table,
|
/// Options already set on the connection will be inherited by the table,
|
||||||
/// but can be overridden here.
|
/// but can be overridden here.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
let store_params = self
|
let store_params = self
|
||||||
.request
|
.request
|
||||||
@@ -188,7 +188,7 @@ impl OpenTableBuilder {
|
|||||||
/// Options already set on the connection will be inherited by the table,
|
/// Options already set on the connection will be inherited by the table,
|
||||||
/// but can be overridden here.
|
/// but can be overridden here.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_options(
|
pub fn storage_options(
|
||||||
mut self,
|
mut self,
|
||||||
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
@@ -582,6 +582,14 @@ pub struct ConnectRequest {
|
|||||||
/// Database specific options
|
/// Database specific options
|
||||||
pub options: HashMap<String, String>,
|
pub options: HashMap<String, String>,
|
||||||
|
|
||||||
|
/// Extra properties for the equivalent namespace client.
|
||||||
|
///
|
||||||
|
/// For a local [`ListingDatabase`], these are merged into the backing
|
||||||
|
/// `DirectoryNamespace` properties. This is useful for namespace-specific
|
||||||
|
/// settings such as `table_version_tracking_enabled` that are distinct from
|
||||||
|
/// storage options.
|
||||||
|
pub namespace_client_properties: HashMap<String, String>,
|
||||||
|
|
||||||
/// The interval at which to check for updates from other processes.
|
/// The interval at which to check for updates from other processes.
|
||||||
///
|
///
|
||||||
/// If None, then consistency is not checked. For performance
|
/// If None, then consistency is not checked. For performance
|
||||||
@@ -621,6 +629,7 @@ impl ConnectBuilder {
|
|||||||
client_config: Default::default(),
|
client_config: Default::default(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
options: HashMap::new(),
|
options: HashMap::new(),
|
||||||
|
namespace_client_properties: HashMap::new(),
|
||||||
session: None,
|
session: None,
|
||||||
},
|
},
|
||||||
embedding_registry: None,
|
embedding_registry: None,
|
||||||
@@ -738,7 +747,7 @@ impl ConnectBuilder {
|
|||||||
|
|
||||||
/// Set an option for the storage layer.
|
/// Set an option for the storage layer.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
self.request.options.insert(key.into(), value.into());
|
self.request.options.insert(key.into(), value.into());
|
||||||
self
|
self
|
||||||
@@ -746,7 +755,7 @@ impl ConnectBuilder {
|
|||||||
|
|
||||||
/// Set multiple options for the storage layer.
|
/// Set multiple options for the storage layer.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_options(
|
pub fn storage_options(
|
||||||
mut self,
|
mut self,
|
||||||
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
@@ -757,6 +766,31 @@ impl ConnectBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Set an additional property for the equivalent namespace client.
|
||||||
|
pub fn namespace_client_property(
|
||||||
|
mut self,
|
||||||
|
key: impl Into<String>,
|
||||||
|
value: impl Into<String>,
|
||||||
|
) -> Self {
|
||||||
|
self.request
|
||||||
|
.namespace_client_properties
|
||||||
|
.insert(key.into(), value.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set multiple additional properties for the equivalent namespace client.
|
||||||
|
pub fn namespace_client_properties(
|
||||||
|
mut self,
|
||||||
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
|
) -> Self {
|
||||||
|
for (key, value) in pairs {
|
||||||
|
self.request
|
||||||
|
.namespace_client_properties
|
||||||
|
.insert(key.into(), value.into());
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// The interval at which to check for updates from other processes. This
|
/// The interval at which to check for updates from other processes. This
|
||||||
/// only affects LanceDB OSS.
|
/// only affects LanceDB OSS.
|
||||||
///
|
///
|
||||||
@@ -881,7 +915,7 @@ use std::collections::HashSet;
|
|||||||
/// These operations will be executed on the namespace server instead of locally
|
/// These operations will be executed on the namespace server instead of locally
|
||||||
/// when enabled via [`ConnectNamespaceBuilder::pushdown_operations`].
|
/// when enabled via [`ConnectNamespaceBuilder::pushdown_operations`].
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||||
pub enum PushdownOperation {
|
pub enum NamespaceClientPushdownOperation {
|
||||||
/// Execute queries on the namespace server via `query_table()` instead of locally.
|
/// Execute queries on the namespace server via `query_table()` instead of locally.
|
||||||
QueryTable,
|
QueryTable,
|
||||||
/// Execute table creation on the namespace server via `create_table()`
|
/// Execute table creation on the namespace server via `create_table()`
|
||||||
@@ -893,10 +927,11 @@ pub struct ConnectNamespaceBuilder {
|
|||||||
ns_impl: String,
|
ns_impl: String,
|
||||||
properties: HashMap<String, String>,
|
properties: HashMap<String, String>,
|
||||||
storage_options: HashMap<String, String>,
|
storage_options: HashMap<String, String>,
|
||||||
|
namespace_client_properties: HashMap<String, String>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
|
embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
|
||||||
session: Option<Arc<lance::session::Session>>,
|
session: Option<Arc<lance::session::Session>>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ConnectNamespaceBuilder {
|
impl ConnectNamespaceBuilder {
|
||||||
@@ -905,6 +940,7 @@ impl ConnectNamespaceBuilder {
|
|||||||
ns_impl: ns_impl.to_string(),
|
ns_impl: ns_impl.to_string(),
|
||||||
properties,
|
properties,
|
||||||
storage_options: HashMap::new(),
|
storage_options: HashMap::new(),
|
||||||
|
namespace_client_properties: HashMap::new(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
embedding_registry: None,
|
embedding_registry: None,
|
||||||
session: None,
|
session: None,
|
||||||
@@ -914,7 +950,7 @@ impl ConnectNamespaceBuilder {
|
|||||||
|
|
||||||
/// Set an option for the storage layer.
|
/// Set an option for the storage layer.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
self.storage_options.insert(key.into(), value.into());
|
self.storage_options.insert(key.into(), value.into());
|
||||||
self
|
self
|
||||||
@@ -922,7 +958,7 @@ impl ConnectNamespaceBuilder {
|
|||||||
|
|
||||||
/// Set multiple options for the storage layer.
|
/// Set multiple options for the storage layer.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_options(
|
pub fn storage_options(
|
||||||
mut self,
|
mut self,
|
||||||
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
@@ -933,6 +969,29 @@ impl ConnectNamespaceBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Set an additional namespace client property.
|
||||||
|
pub fn namespace_client_property(
|
||||||
|
mut self,
|
||||||
|
key: impl Into<String>,
|
||||||
|
value: impl Into<String>,
|
||||||
|
) -> Self {
|
||||||
|
self.namespace_client_properties
|
||||||
|
.insert(key.into(), value.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Set multiple additional namespace client properties.
|
||||||
|
pub fn namespace_client_properties(
|
||||||
|
mut self,
|
||||||
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
|
) -> Self {
|
||||||
|
for (key, value) in pairs {
|
||||||
|
self.namespace_client_properties
|
||||||
|
.insert(key.into(), value.into());
|
||||||
|
}
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// The interval at which to check for updates from other processes.
|
/// The interval at which to check for updates from other processes.
|
||||||
///
|
///
|
||||||
/// If left unset, consistency is not checked. For maximum read
|
/// If left unset, consistency is not checked. For maximum read
|
||||||
@@ -970,11 +1029,11 @@ impl ConnectNamespaceBuilder {
|
|||||||
/// and leveraging server-side compute resources.
|
/// and leveraging server-side compute resources.
|
||||||
///
|
///
|
||||||
/// Available operations:
|
/// Available operations:
|
||||||
/// - [`PushdownOperation::QueryTable`]: Execute queries via `namespace.query_table()`
|
/// - [`NamespaceClientPushdownOperation::QueryTable`]: Execute queries via `namespace.query_table()`
|
||||||
/// - [`PushdownOperation::CreateTable`]: Execute table creation via `namespace.create_table()`
|
/// - [`NamespaceClientPushdownOperation::CreateTable`]: Execute table creation via `namespace.create_table()`
|
||||||
///
|
///
|
||||||
/// By default, no operations are pushed down (all executed locally).
|
/// By default, no operations are pushed down (all executed locally).
|
||||||
pub fn pushdown_operation(mut self, operation: PushdownOperation) -> Self {
|
pub fn pushdown_operation(mut self, operation: NamespaceClientPushdownOperation) -> Self {
|
||||||
self.pushdown_operations.insert(operation);
|
self.pushdown_operations.insert(operation);
|
||||||
self
|
self
|
||||||
}
|
}
|
||||||
@@ -984,7 +1043,7 @@ impl ConnectNamespaceBuilder {
|
|||||||
/// See [`Self::pushdown_operation`] for details.
|
/// See [`Self::pushdown_operation`] for details.
|
||||||
pub fn pushdown_operations(
|
pub fn pushdown_operations(
|
||||||
mut self,
|
mut self,
|
||||||
operations: impl IntoIterator<Item = PushdownOperation>,
|
operations: impl IntoIterator<Item = NamespaceClientPushdownOperation>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
self.pushdown_operations.extend(operations);
|
self.pushdown_operations.extend(operations);
|
||||||
self
|
self
|
||||||
@@ -994,10 +1053,13 @@ impl ConnectNamespaceBuilder {
|
|||||||
pub async fn execute(self) -> Result<Connection> {
|
pub async fn execute(self) -> Result<Connection> {
|
||||||
use crate::database::namespace::LanceNamespaceDatabase;
|
use crate::database::namespace::LanceNamespaceDatabase;
|
||||||
|
|
||||||
|
let mut properties = self.properties;
|
||||||
|
properties.extend(self.namespace_client_properties);
|
||||||
|
|
||||||
let internal = Arc::new(
|
let internal = Arc::new(
|
||||||
LanceNamespaceDatabase::connect(
|
LanceNamespaceDatabase::connect(
|
||||||
&self.ns_impl,
|
&self.ns_impl,
|
||||||
self.properties,
|
properties,
|
||||||
self.storage_options,
|
self.storage_options,
|
||||||
self.read_consistency_interval,
|
self.read_consistency_interval,
|
||||||
self.session,
|
self.session,
|
||||||
@@ -1117,6 +1179,31 @@ mod tests {
|
|||||||
assert_eq!(db.uri(), relative_uri.to_str().unwrap().to_string());
|
assert_eq!(db.uri(), relative_uri.to_str().unwrap().to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_connect_with_namespace_client_properties() {
|
||||||
|
let tmp_dir = tempdir().unwrap();
|
||||||
|
let uri = tmp_dir.path().to_str().unwrap();
|
||||||
|
|
||||||
|
let db = connect(uri)
|
||||||
|
.namespace_client_property("table_version_tracking_enabled", "true")
|
||||||
|
.namespace_client_property("manifest_enabled", "true")
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let (ns_impl, properties) = db.namespace_client_config().await.unwrap();
|
||||||
|
assert_eq!(ns_impl, "dir");
|
||||||
|
assert_eq!(properties.get("root"), Some(&uri.to_string()));
|
||||||
|
assert_eq!(
|
||||||
|
properties.get("table_version_tracking_enabled"),
|
||||||
|
Some(&"true".to_string())
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
properties.get("manifest_enabled"),
|
||||||
|
Some(&"true".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_table_names() {
|
async fn test_table_names() {
|
||||||
let tc = new_test_connection().await.unwrap();
|
let tc = new_test_connection().await.unwrap();
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ impl CreateTableBuilder {
|
|||||||
/// Options already set on the connection will be inherited by the table,
|
/// Options already set on the connection will be inherited by the table,
|
||||||
/// but can be overridden here.
|
/// but can be overridden here.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
let store_params = self
|
let store_params = self
|
||||||
.request
|
.request
|
||||||
@@ -73,7 +73,7 @@ impl CreateTableBuilder {
|
|||||||
/// Options already set on the connection will be inherited by the table,
|
/// Options already set on the connection will be inherited by the table,
|
||||||
/// but can be overridden here.
|
/// but can be overridden here.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_options(
|
pub fn storage_options(
|
||||||
mut self,
|
mut self,
|
||||||
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ use snafu::ResultExt;
|
|||||||
|
|
||||||
use crate::connection::ConnectRequest;
|
use crate::connection::ConnectRequest;
|
||||||
use crate::database::ReadConsistency;
|
use crate::database::ReadConsistency;
|
||||||
|
use crate::database::namespace::LanceNamespaceDatabase;
|
||||||
use crate::error::{CreateDirSnafu, Error, Result};
|
use crate::error::{CreateDirSnafu, Error, Result};
|
||||||
use crate::io::object_store::MirroringObjectStoreWrapper;
|
use crate::io::object_store::MirroringObjectStoreWrapper;
|
||||||
use crate::table::NativeTable;
|
use crate::table::NativeTable;
|
||||||
@@ -73,7 +74,7 @@ pub struct ListingDatabaseOptions {
|
|||||||
/// These are used to create/list tables and they are inherited by all tables
|
/// These are used to create/list tables and they are inherited by all tables
|
||||||
/// opened by this database.
|
/// opened by this database.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub storage_options: HashMap<String, String>,
|
pub storage_options: HashMap<String, String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -185,7 +186,7 @@ impl ListingDatabaseOptionsBuilder {
|
|||||||
|
|
||||||
/// Set an option for the storage layer.
|
/// Set an option for the storage layer.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
|
||||||
self.options
|
self.options
|
||||||
.storage_options
|
.storage_options
|
||||||
@@ -195,7 +196,7 @@ impl ListingDatabaseOptionsBuilder {
|
|||||||
|
|
||||||
/// Set multiple options for the storage layer.
|
/// Set multiple options for the storage layer.
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
pub fn storage_options(
|
pub fn storage_options(
|
||||||
mut self,
|
mut self,
|
||||||
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
|
||||||
@@ -255,6 +256,9 @@ pub struct ListingDatabase {
|
|||||||
|
|
||||||
// Session for object stores and caching
|
// Session for object stores and caching
|
||||||
session: Arc<lance::session::Session>,
|
session: Arc<lance::session::Session>,
|
||||||
|
|
||||||
|
// Namespace-backed database for child namespace operations
|
||||||
|
namespace_database: Arc<LanceNamespaceDatabase>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Display for ListingDatabase {
|
impl std::fmt::Display for ListingDatabase {
|
||||||
@@ -281,6 +285,44 @@ const MIRRORED_STORE: &str = "mirroredStore";
|
|||||||
|
|
||||||
/// A connection to LanceDB
|
/// A connection to LanceDB
|
||||||
impl ListingDatabase {
|
impl ListingDatabase {
|
||||||
|
fn build_namespace_client_properties(
|
||||||
|
uri: &str,
|
||||||
|
storage_options: &HashMap<String, String>,
|
||||||
|
namespace_client_properties: HashMap<String, String>,
|
||||||
|
) -> HashMap<String, String> {
|
||||||
|
let mut properties = namespace_client_properties;
|
||||||
|
properties.insert("root".to_string(), uri.to_string());
|
||||||
|
for (key, value) in storage_options {
|
||||||
|
properties.insert(format!("storage.{}", key), value.clone());
|
||||||
|
}
|
||||||
|
properties
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn connect_namespace_database(
|
||||||
|
uri: &str,
|
||||||
|
storage_options: HashMap<String, String>,
|
||||||
|
namespace_client_properties: HashMap<String, String>,
|
||||||
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
|
session: Arc<lance::session::Session>,
|
||||||
|
) -> Result<Arc<LanceNamespaceDatabase>> {
|
||||||
|
let ns_properties = Self::build_namespace_client_properties(
|
||||||
|
uri,
|
||||||
|
&storage_options,
|
||||||
|
namespace_client_properties,
|
||||||
|
);
|
||||||
|
Ok(Arc::new(
|
||||||
|
LanceNamespaceDatabase::connect(
|
||||||
|
"dir",
|
||||||
|
ns_properties,
|
||||||
|
storage_options,
|
||||||
|
read_consistency_interval,
|
||||||
|
Some(session),
|
||||||
|
HashSet::new(),
|
||||||
|
)
|
||||||
|
.await?,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
/// Connect to a listing database
|
/// Connect to a listing database
|
||||||
///
|
///
|
||||||
/// The URI should be a path to a directory where the tables are stored.
|
/// The URI should be a path to a directory where the tables are stored.
|
||||||
@@ -300,6 +342,7 @@ impl ListingDatabase {
|
|||||||
uri,
|
uri,
|
||||||
request.read_consistency_interval,
|
request.read_consistency_interval,
|
||||||
options.new_table_config,
|
options.new_table_config,
|
||||||
|
request.namespace_client_properties.clone(),
|
||||||
request.session.clone(),
|
request.session.clone(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -387,6 +430,15 @@ impl ListingDatabase {
|
|||||||
None => None,
|
None => None,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let namespace_database = Self::connect_namespace_database(
|
||||||
|
&table_base_uri,
|
||||||
|
options.storage_options.clone(),
|
||||||
|
request.namespace_client_properties.clone(),
|
||||||
|
request.read_consistency_interval,
|
||||||
|
session.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
uri: table_base_uri,
|
uri: table_base_uri,
|
||||||
query_string,
|
query_string,
|
||||||
@@ -398,6 +450,7 @@ impl ListingDatabase {
|
|||||||
storage_options_provider: None,
|
storage_options_provider: None,
|
||||||
new_table_config: options.new_table_config,
|
new_table_config: options.new_table_config,
|
||||||
session,
|
session,
|
||||||
|
namespace_database,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
@@ -405,6 +458,7 @@ impl ListingDatabase {
|
|||||||
uri,
|
uri,
|
||||||
request.read_consistency_interval,
|
request.read_consistency_interval,
|
||||||
options.new_table_config,
|
options.new_table_config,
|
||||||
|
request.namespace_client_properties.clone(),
|
||||||
request.session.clone(),
|
request.session.clone(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -416,6 +470,7 @@ impl ListingDatabase {
|
|||||||
path: &str,
|
path: &str,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
new_table_config: NewTableConfig,
|
new_table_config: NewTableConfig,
|
||||||
|
namespace_client_properties: HashMap<String, String>,
|
||||||
session: Option<Arc<lance::session::Session>>,
|
session: Option<Arc<lance::session::Session>>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let session = session.unwrap_or_else(|| Arc::new(lance::session::Session::default()));
|
let session = session.unwrap_or_else(|| Arc::new(lance::session::Session::default()));
|
||||||
@@ -429,6 +484,15 @@ impl ListingDatabase {
|
|||||||
Self::try_create_dir(path).context(CreateDirSnafu { path })?;
|
Self::try_create_dir(path).context(CreateDirSnafu { path })?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let namespace_database = Self::connect_namespace_database(
|
||||||
|
path,
|
||||||
|
HashMap::new(),
|
||||||
|
namespace_client_properties,
|
||||||
|
read_consistency_interval,
|
||||||
|
session.clone(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
uri: path.to_string(),
|
uri: path.to_string(),
|
||||||
query_string: None,
|
query_string: None,
|
||||||
@@ -440,6 +504,7 @@ impl ListingDatabase {
|
|||||||
storage_options_provider: None,
|
storage_options_provider: None,
|
||||||
new_table_config,
|
new_table_config,
|
||||||
session,
|
session,
|
||||||
|
namespace_database,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -497,6 +562,10 @@ impl ListingDatabase {
|
|||||||
Ok(uri)
|
Ok(uri)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn namespace_database(&self) -> Arc<LanceNamespaceDatabase> {
|
||||||
|
self.namespace_database.clone()
|
||||||
|
}
|
||||||
|
|
||||||
async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
|
async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
|
||||||
let object_store_params = ObjectStoreParams {
|
let object_store_params = ObjectStoreParams {
|
||||||
storage_options_accessor: if self.storage_options.is_empty() {
|
storage_options_accessor: if self.storage_options.is_empty() {
|
||||||
@@ -696,16 +765,7 @@ impl Database for ListingDatabase {
|
|||||||
&self,
|
&self,
|
||||||
request: ListNamespacesRequest,
|
request: ListNamespacesRequest,
|
||||||
) -> Result<ListNamespacesResponse> {
|
) -> Result<ListNamespacesResponse> {
|
||||||
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
|
self.namespace_database().list_namespaces(request).await
|
||||||
return Err(Error::NotSupported {
|
|
||||||
message: "Namespace operations are not supported for listing database".into(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(ListNamespacesResponse {
|
|
||||||
namespaces: Vec::new(),
|
|
||||||
page_token: None,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn uri(&self) -> &str {
|
fn uri(&self) -> &str {
|
||||||
@@ -726,36 +786,26 @@ impl Database for ListingDatabase {
|
|||||||
|
|
||||||
async fn create_namespace(
|
async fn create_namespace(
|
||||||
&self,
|
&self,
|
||||||
_request: CreateNamespaceRequest,
|
request: CreateNamespaceRequest,
|
||||||
) -> Result<CreateNamespaceResponse> {
|
) -> Result<CreateNamespaceResponse> {
|
||||||
Err(Error::NotSupported {
|
self.namespace_database().create_namespace(request).await
|
||||||
message: "Namespace operations are not supported for listing database".into(),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn drop_namespace(
|
async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
|
||||||
&self,
|
self.namespace_database().drop_namespace(request).await
|
||||||
_request: DropNamespaceRequest,
|
|
||||||
) -> Result<DropNamespaceResponse> {
|
|
||||||
Err(Error::NotSupported {
|
|
||||||
message: "Namespace operations are not supported for listing database".into(),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn describe_namespace(
|
async fn describe_namespace(
|
||||||
&self,
|
&self,
|
||||||
_request: DescribeNamespaceRequest,
|
request: DescribeNamespaceRequest,
|
||||||
) -> Result<DescribeNamespaceResponse> {
|
) -> Result<DescribeNamespaceResponse> {
|
||||||
Err(Error::NotSupported {
|
self.namespace_database().describe_namespace(request).await
|
||||||
message: "Namespace operations are not supported for listing database".into(),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[allow(deprecated)]
|
||||||
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
|
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
|
||||||
if !request.namespace_path.is_empty() {
|
if !request.namespace_path.is_empty() {
|
||||||
return Err(Error::NotSupported {
|
return self.namespace_database().table_names(request).await;
|
||||||
message: "Namespace parameter is not supported for listing database. Only root namespace is supported.".into(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
let mut f = self
|
let mut f = self
|
||||||
.object_store
|
.object_store
|
||||||
@@ -788,9 +838,7 @@ impl Database for ListingDatabase {
|
|||||||
|
|
||||||
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
|
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
|
||||||
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
|
if request.id.as_ref().map(|v| !v.is_empty()).unwrap_or(false) {
|
||||||
return Err(Error::NotSupported {
|
return self.namespace_database().list_tables(request).await;
|
||||||
message: "Namespace parameter is not supported for listing database. Only root namespace is supported.".into(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
let mut f = self
|
let mut f = self
|
||||||
.object_store
|
.object_store
|
||||||
@@ -838,11 +886,8 @@ impl Database for ListingDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn create_table(&self, request: CreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn create_table(&self, request: CreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
// When namespace is not empty, location must be provided
|
if !request.namespace_path.is_empty() {
|
||||||
if !request.namespace_path.is_empty() && request.location.is_none() {
|
return self.namespace_database().create_table(request).await;
|
||||||
return Err(Error::InvalidInput {
|
|
||||||
message: "Location must be provided when namespace is not empty".into(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
// Use provided location if available, otherwise derive from table name
|
// Use provided location if available, otherwise derive from table name
|
||||||
let table_uri = request
|
let table_uri = request
|
||||||
@@ -959,11 +1004,8 @@ impl Database for ListingDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn open_table(&self, mut request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn open_table(&self, mut request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
// When namespace is not empty, location must be provided
|
if !request.namespace_path.is_empty() {
|
||||||
if !request.namespace_path.is_empty() && request.location.is_none() {
|
return self.namespace_database().open_table(request).await;
|
||||||
return Err(Error::InvalidInput {
|
|
||||||
message: "Location must be provided when namespace is not empty".into(),
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
// Use provided location if available, otherwise derive from table name
|
// Use provided location if available, otherwise derive from table name
|
||||||
let table_uri = request
|
let table_uri = request
|
||||||
@@ -1059,9 +1101,10 @@ impl Database for ListingDatabase {
|
|||||||
|
|
||||||
async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> {
|
async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> {
|
||||||
if !namespace_path.is_empty() {
|
if !namespace_path.is_empty() {
|
||||||
return Err(Error::NotSupported {
|
return self
|
||||||
message: "Namespace parameter is not supported for listing database.".into(),
|
.namespace_database()
|
||||||
});
|
.drop_table(name, namespace_path)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
self.drop_tables(vec![name.to_string()]).await
|
self.drop_tables(vec![name.to_string()]).await
|
||||||
}
|
}
|
||||||
@@ -1070,9 +1113,10 @@ impl Database for ListingDatabase {
|
|||||||
async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()> {
|
async fn drop_all_tables(&self, namespace_path: &[String]) -> Result<()> {
|
||||||
// Check if namespace parameter is provided
|
// Check if namespace parameter is provided
|
||||||
if !namespace_path.is_empty() {
|
if !namespace_path.is_empty() {
|
||||||
return Err(Error::NotSupported {
|
return self
|
||||||
message: "Namespace parameter is not supported for listing database.".into(),
|
.namespace_database()
|
||||||
});
|
.drop_all_tables(namespace_path)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
let tables = self.table_names(TableNamesRequest::default()).await?;
|
let tables = self.table_names(TableNamesRequest::default()).await?;
|
||||||
self.drop_tables(tables).await
|
self.drop_tables(tables).await
|
||||||
@@ -1083,30 +1127,11 @@ impl Database for ListingDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> {
|
async fn namespace_client(&self) -> Result<Arc<dyn lance_namespace::LanceNamespace>> {
|
||||||
// Create a DirectoryNamespace pointing to the same root with the same storage options
|
self.namespace_database.namespace_client().await
|
||||||
let mut builder = lance_namespace_impls::DirectoryNamespaceBuilder::new(&self.uri);
|
|
||||||
|
|
||||||
// Add storage options
|
|
||||||
if !self.storage_options.is_empty() {
|
|
||||||
builder = builder.storage_options(self.storage_options.clone());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use the same session
|
|
||||||
builder = builder.session(self.session.clone());
|
|
||||||
|
|
||||||
let namespace = builder.build().await.map_err(|e| Error::Runtime {
|
|
||||||
message: format!("Failed to create namespace client: {}", e),
|
|
||||||
})?;
|
|
||||||
Ok(Arc::new(namespace) as Arc<dyn lance_namespace::LanceNamespace>)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn namespace_client_config(&self) -> Result<(String, HashMap<String, String>)> {
|
async fn namespace_client_config(&self) -> Result<(String, HashMap<String, String>)> {
|
||||||
let mut properties = HashMap::new();
|
self.namespace_database.namespace_client_config().await
|
||||||
properties.insert("root".to_string(), self.uri.clone());
|
|
||||||
for (key, value) in &self.storage_options {
|
|
||||||
properties.insert(format!("storage.{}", key), value.clone());
|
|
||||||
}
|
|
||||||
Ok(("dir".to_string(), properties))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1132,6 +1157,7 @@ mod tests {
|
|||||||
#[cfg(feature = "remote")]
|
#[cfg(feature = "remote")]
|
||||||
client_config: Default::default(),
|
client_config: Default::default(),
|
||||||
options: Default::default(),
|
options: Default::default(),
|
||||||
|
namespace_client_properties: Default::default(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
session: None,
|
session: None,
|
||||||
};
|
};
|
||||||
@@ -1265,6 +1291,7 @@ mod tests {
|
|||||||
#[cfg(feature = "remote")]
|
#[cfg(feature = "remote")]
|
||||||
client_config: Default::default(),
|
client_config: Default::default(),
|
||||||
options: options.clone(),
|
options: options.clone(),
|
||||||
|
namespace_client_properties: Default::default(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
session: None,
|
session: None,
|
||||||
};
|
};
|
||||||
@@ -1799,6 +1826,7 @@ mod tests {
|
|||||||
#[cfg(feature = "remote")]
|
#[cfg(feature = "remote")]
|
||||||
client_config: Default::default(),
|
client_config: Default::default(),
|
||||||
options,
|
options,
|
||||||
|
namespace_client_properties: Default::default(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
session: None,
|
session: None,
|
||||||
};
|
};
|
||||||
@@ -1904,6 +1932,7 @@ mod tests {
|
|||||||
#[cfg(feature = "remote")]
|
#[cfg(feature = "remote")]
|
||||||
client_config: Default::default(),
|
client_config: Default::default(),
|
||||||
options,
|
options,
|
||||||
|
namespace_client_properties: Default::default(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
session: None,
|
session: None,
|
||||||
};
|
};
|
||||||
@@ -1975,6 +2004,7 @@ mod tests {
|
|||||||
#[cfg(feature = "remote")]
|
#[cfg(feature = "remote")]
|
||||||
client_config: Default::default(),
|
client_config: Default::default(),
|
||||||
options,
|
options,
|
||||||
|
namespace_client_properties: Default::default(),
|
||||||
read_consistency_interval: None,
|
read_consistency_interval: None,
|
||||||
session: None,
|
session: None,
|
||||||
};
|
};
|
||||||
@@ -2108,4 +2138,209 @@ mod tests {
|
|||||||
assert!(tables.contains(&"table1".to_string()));
|
assert!(tables.contains(&"table1".to_string()));
|
||||||
assert!(tables.contains(&"table2".to_string()));
|
assert!(tables.contains(&"table2".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_listing_database_namespace_operations() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
|
||||||
|
db.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(vec!["parent".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
db.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(vec!["parent".to_string(), "child".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let root_namespaces = db
|
||||||
|
.list_namespaces(ListNamespacesRequest {
|
||||||
|
id: Some(vec![]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(root_namespaces.namespaces.contains(&"parent".to_string()));
|
||||||
|
|
||||||
|
let child_namespaces = db
|
||||||
|
.list_namespaces(ListNamespacesRequest {
|
||||||
|
id: Some(vec!["parent".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(child_namespaces.namespaces.contains(&"child".to_string()));
|
||||||
|
|
||||||
|
db.describe_namespace(DescribeNamespaceRequest {
|
||||||
|
id: Some(vec!["parent".to_string(), "child".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[cfg(not(windows))] // TODO: support Windows once directory namespace-backed listing DB tests are supported.
|
||||||
|
async fn test_listing_database_with_namespace_client_properties() {
|
||||||
|
let tempdir = tempdir().unwrap();
|
||||||
|
let uri = tempdir.path().to_str().unwrap();
|
||||||
|
|
||||||
|
let mut namespace_client_properties = HashMap::new();
|
||||||
|
namespace_client_properties.insert(
|
||||||
|
"table_version_tracking_enabled".to_string(),
|
||||||
|
"true".to_string(),
|
||||||
|
);
|
||||||
|
namespace_client_properties.insert("manifest_enabled".to_string(), "true".to_string());
|
||||||
|
|
||||||
|
let request = ConnectRequest {
|
||||||
|
uri: uri.to_string(),
|
||||||
|
#[cfg(feature = "remote")]
|
||||||
|
client_config: Default::default(),
|
||||||
|
options: Default::default(),
|
||||||
|
namespace_client_properties,
|
||||||
|
read_consistency_interval: None,
|
||||||
|
session: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let db = ListingDatabase::connect_with_options(&request)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let namespace_path = vec!["test_ns".to_string()];
|
||||||
|
|
||||||
|
db.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(namespace_path.clone()),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("name", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "managed_table".to_string(),
|
||||||
|
namespace_path: namespace_path.clone(),
|
||||||
|
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
location: None,
|
||||||
|
namespace_client: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let namespace_client = db.namespace_client().await.unwrap();
|
||||||
|
let describe = namespace_client
|
||||||
|
.describe_table(lance_namespace::models::DescribeTableRequest {
|
||||||
|
id: Some(vec!["test_ns".to_string(), "managed_table".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(describe.managed_versioning, Some(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_listing_database_nested_namespace_table_ops() {
|
||||||
|
let (_tempdir, db) = setup_database().await;
|
||||||
|
let namespace_path = vec!["parent".to_string(), "child".to_string()];
|
||||||
|
|
||||||
|
db.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(vec!["parent".to_string()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
db.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(namespace_path.clone()),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let schema = Arc::new(Schema::new(vec![
|
||||||
|
Field::new("id", DataType::Int32, false),
|
||||||
|
Field::new("name", DataType::Utf8, false),
|
||||||
|
]));
|
||||||
|
|
||||||
|
db.create_table(CreateTableRequest {
|
||||||
|
name: "nested_table".to_string(),
|
||||||
|
namespace_path: namespace_path.clone(),
|
||||||
|
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
|
||||||
|
mode: CreateTableMode::Create,
|
||||||
|
write_options: Default::default(),
|
||||||
|
location: None,
|
||||||
|
namespace_client: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let namespace_client = db.namespace_client().await.unwrap();
|
||||||
|
let describe = namespace_client
|
||||||
|
.describe_table(lance_namespace::models::DescribeTableRequest {
|
||||||
|
id: Some(vec![
|
||||||
|
"parent".to_string(),
|
||||||
|
"child".to_string(),
|
||||||
|
"nested_table".to_string(),
|
||||||
|
]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(describe.location.is_some());
|
||||||
|
|
||||||
|
let table = db
|
||||||
|
.open_table(OpenTableRequest {
|
||||||
|
name: "nested_table".to_string(),
|
||||||
|
namespace_path: namespace_path.clone(),
|
||||||
|
index_cache_size: None,
|
||||||
|
lance_read_params: None,
|
||||||
|
location: None,
|
||||||
|
namespace_client: None,
|
||||||
|
managed_versioning: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table.name(), "nested_table");
|
||||||
|
|
||||||
|
#[allow(deprecated)]
|
||||||
|
let table_names = db
|
||||||
|
.table_names(TableNamesRequest {
|
||||||
|
namespace_path: namespace_path.clone(),
|
||||||
|
start_after: None,
|
||||||
|
limit: None,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(table_names, vec!["nested_table".to_string()]);
|
||||||
|
|
||||||
|
let list_tables = db
|
||||||
|
.list_tables(ListTablesRequest {
|
||||||
|
id: Some(namespace_path.clone()),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(list_tables.tables, vec!["nested_table".to_string()]);
|
||||||
|
|
||||||
|
db.drop_table("nested_table", &namespace_path)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let post_drop = db
|
||||||
|
.list_tables(ListTablesRequest {
|
||||||
|
id: Some(namespace_path),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(post_drop.tables.is_empty());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,10 +22,11 @@ use lance_namespace_impls::ConnectBuilder;
|
|||||||
use lance_table::io::commit::CommitHandler;
|
use lance_table::io::commit::CommitHandler;
|
||||||
use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
|
use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
|
||||||
|
|
||||||
use crate::connection::PushdownOperation;
|
use crate::connection::NamespaceClientPushdownOperation;
|
||||||
use crate::database::ReadConsistency;
|
use crate::database::ReadConsistency;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::table::NativeTable;
|
use crate::table::NativeTable;
|
||||||
|
use lance::dataset::WriteMode;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
BaseTable, CloneTableRequest, CreateTableMode, CreateTableRequest as DbCreateTableRequest,
|
BaseTable, CloneTableRequest, CreateTableMode, CreateTableRequest as DbCreateTableRequest,
|
||||||
@@ -44,7 +45,7 @@ pub struct LanceNamespaceDatabase {
|
|||||||
// database URI
|
// database URI
|
||||||
uri: String,
|
uri: String,
|
||||||
// Operations to push down to the namespace server
|
// Operations to push down to the namespace server
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
// Namespace implementation type (e.g., "dir", "rest")
|
// Namespace implementation type (e.g., "dir", "rest")
|
||||||
ns_impl: String,
|
ns_impl: String,
|
||||||
// Namespace properties used to construct the namespace client
|
// Namespace properties used to construct the namespace client
|
||||||
@@ -52,13 +53,34 @@ pub struct LanceNamespaceDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl LanceNamespaceDatabase {
|
impl LanceNamespaceDatabase {
|
||||||
|
pub fn from_namespace_client(
|
||||||
|
namespace_client: Arc<dyn LanceNamespace>,
|
||||||
|
namespace_client_impl: String,
|
||||||
|
namespace_client_properties: HashMap<String, String>,
|
||||||
|
storage_options: HashMap<String, String>,
|
||||||
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
|
session: Option<Arc<lance::session::Session>>,
|
||||||
|
namespace_client_pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
namespace: namespace_client,
|
||||||
|
storage_options,
|
||||||
|
read_consistency_interval,
|
||||||
|
session,
|
||||||
|
uri: format!("namespace://{}", namespace_client_impl),
|
||||||
|
pushdown_operations: namespace_client_pushdown_operations,
|
||||||
|
ns_impl: namespace_client_impl,
|
||||||
|
ns_properties: namespace_client_properties,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn connect(
|
pub async fn connect(
|
||||||
ns_impl: &str,
|
ns_impl: &str,
|
||||||
ns_properties: HashMap<String, String>,
|
ns_properties: HashMap<String, String>,
|
||||||
storage_options: HashMap<String, String>,
|
storage_options: HashMap<String, String>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
session: Option<Arc<lance::session::Session>>,
|
session: Option<Arc<lance::session::Session>>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let mut builder = ConnectBuilder::new(ns_impl);
|
let mut builder = ConnectBuilder::new(ns_impl);
|
||||||
for (key, value) in ns_properties.clone() {
|
for (key, value) in ns_properties.clone() {
|
||||||
@@ -163,37 +185,23 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
|
||||||
let mut table_id = request.namespace_path.clone();
|
let mut table_id = request.namespace_path.clone();
|
||||||
table_id.push(request.name.clone());
|
table_id.push(request.name.clone());
|
||||||
let describe_request = DescribeTableRequest {
|
let mut existing_table = None;
|
||||||
id: Some(table_id.clone()),
|
|
||||||
..Default::default()
|
|
||||||
};
|
|
||||||
|
|
||||||
let describe_result = self.namespace.describe_table(describe_request).await;
|
|
||||||
|
|
||||||
match request.mode {
|
match request.mode {
|
||||||
CreateTableMode::Create => {
|
CreateTableMode::Create => {}
|
||||||
if describe_result.is_ok() {
|
|
||||||
return Err(Error::TableAlreadyExists {
|
|
||||||
name: request.name.clone(),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
CreateTableMode::Overwrite => {
|
CreateTableMode::Overwrite => {
|
||||||
if describe_result.is_ok() {
|
let describe_request = DescribeTableRequest {
|
||||||
// Drop the existing table - must succeed
|
id: Some(table_id.clone()),
|
||||||
let drop_request = DropTableRequest {
|
..Default::default()
|
||||||
id: Some(table_id.clone()),
|
};
|
||||||
..Default::default()
|
existing_table = self.namespace.describe_table(describe_request).await.ok();
|
||||||
};
|
|
||||||
self.namespace
|
|
||||||
.drop_table(drop_request)
|
|
||||||
.await
|
|
||||||
.map_err(|e| Error::Runtime {
|
|
||||||
message: format!("Failed to drop existing table for overwrite: {}", e),
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
CreateTableMode::ExistOk(_) => {
|
CreateTableMode::ExistOk(_) => {
|
||||||
|
let describe_request = DescribeTableRequest {
|
||||||
|
id: Some(table_id.clone()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let describe_result = self.namespace.describe_table(describe_request).await;
|
||||||
if describe_result.is_ok() {
|
if describe_result.is_ok() {
|
||||||
let native_table = NativeTable::open_from_namespace(
|
let native_table = NativeTable::open_from_namespace(
|
||||||
self.namespace.clone(),
|
self.namespace.clone(),
|
||||||
@@ -221,21 +229,82 @@ impl Database for LanceNamespaceDatabase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (location, initial_storage_options, managed_versioning) = {
|
let (location, initial_storage_options, managed_versioning) = {
|
||||||
let response = self.namespace.declare_table(declare_request).await?;
|
if let Some(response) = existing_table {
|
||||||
let loc = response.location.ok_or_else(|| Error::Runtime {
|
let loc = response.location.ok_or_else(|| Error::Runtime {
|
||||||
message: "Table location is missing from declare_table response".to_string(),
|
message: "Table location is missing from describe_table response".to_string(),
|
||||||
})?;
|
})?;
|
||||||
// Use storage options from response, fall back to self.storage_options
|
let opts = response
|
||||||
let opts = response
|
.storage_options
|
||||||
.storage_options
|
.or_else(|| Some(self.storage_options.clone()))
|
||||||
.or_else(|| Some(self.storage_options.clone()))
|
.filter(|o| !o.is_empty());
|
||||||
.filter(|o| !o.is_empty());
|
(loc, opts, response.managed_versioning)
|
||||||
(loc, opts, response.managed_versioning)
|
} else {
|
||||||
|
match self.namespace.declare_table(declare_request).await {
|
||||||
|
Ok(response) => {
|
||||||
|
let loc = response.location.ok_or_else(|| Error::Runtime {
|
||||||
|
message: "Table location is missing from declare_table response"
|
||||||
|
.to_string(),
|
||||||
|
})?;
|
||||||
|
let opts = response
|
||||||
|
.storage_options
|
||||||
|
.or_else(|| Some(self.storage_options.clone()))
|
||||||
|
.filter(|o: &HashMap<String, String>| !o.is_empty());
|
||||||
|
(loc, opts, response.managed_versioning)
|
||||||
|
}
|
||||||
|
Err(e)
|
||||||
|
if matches!(request.mode, CreateTableMode::Create) && {
|
||||||
|
let err_str = e.to_string();
|
||||||
|
err_str.contains("already exists")
|
||||||
|
|| err_str.contains("TableAlreadyExists")
|
||||||
|
|| err_str.contains("table already exists")
|
||||||
|
} =>
|
||||||
|
{
|
||||||
|
let response = self
|
||||||
|
.namespace
|
||||||
|
.describe_table(DescribeTableRequest {
|
||||||
|
id: Some(table_id.clone()),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|describe_err| Error::Runtime {
|
||||||
|
message: format!(
|
||||||
|
"Failed to describe existing declared table after declare conflict: {}",
|
||||||
|
describe_err
|
||||||
|
),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if response.version.is_some() && response.schema.is_some() {
|
||||||
|
return Err(Error::TableAlreadyExists {
|
||||||
|
name: request.name.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let loc = response.location.ok_or_else(|| Error::Runtime {
|
||||||
|
message: "Table location is missing from describe_table response"
|
||||||
|
.to_string(),
|
||||||
|
})?;
|
||||||
|
let opts = response
|
||||||
|
.storage_options
|
||||||
|
.or_else(|| Some(self.storage_options.clone()))
|
||||||
|
.filter(|o: &HashMap<String, String>| !o.is_empty());
|
||||||
|
(loc, opts, response.managed_versioning)
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
return Err(Error::Runtime {
|
||||||
|
message: format!("Failed to declare table: {}", e),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Build write params with storage options and commit handler
|
// Build write params with storage options and commit handler
|
||||||
let mut params = request.write_options.lance_write_params.unwrap_or_default();
|
let mut params = request.write_options.lance_write_params.unwrap_or_default();
|
||||||
|
|
||||||
|
if matches!(request.mode, CreateTableMode::Overwrite) {
|
||||||
|
params.mode = WriteMode::Overwrite;
|
||||||
|
}
|
||||||
|
|
||||||
// Set up storage options if provided
|
// Set up storage options if provided
|
||||||
if let Some(storage_opts) = initial_storage_options {
|
if let Some(storage_opts) = initial_storage_options {
|
||||||
let store_params = params
|
let store_params = params
|
||||||
@@ -450,6 +519,47 @@ mod tests {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_namespace_connection_with_namespace_client_properties() {
|
||||||
|
let tmp_dir = tempdir().unwrap();
|
||||||
|
let root_path = tmp_dir.path().to_str().unwrap().to_string();
|
||||||
|
|
||||||
|
let mut properties = HashMap::new();
|
||||||
|
properties.insert("root".to_string(), root_path);
|
||||||
|
|
||||||
|
let conn = connect_namespace("dir", properties)
|
||||||
|
.namespace_client_property("table_version_tracking_enabled", "true")
|
||||||
|
.namespace_client_property("manifest_enabled", "true")
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.expect("Failed to connect to namespace");
|
||||||
|
|
||||||
|
conn.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(vec!["test_ns".into()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("Failed to create namespace");
|
||||||
|
|
||||||
|
let test_data = create_test_data();
|
||||||
|
conn.create_table("test_table", test_data)
|
||||||
|
.namespace(vec!["test_ns".into()])
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.expect("Failed to create table");
|
||||||
|
|
||||||
|
let namespace_client = conn.namespace_client().await.unwrap();
|
||||||
|
let describe = namespace_client
|
||||||
|
.describe_table(DescribeTableRequest {
|
||||||
|
id: Some(vec!["test_ns".into(), "test_table".into()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("Failed to describe table");
|
||||||
|
|
||||||
|
assert_eq!(describe.managed_versioning, Some(true));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_namespace_create_table_basic() {
|
async fn test_namespace_create_table_basic() {
|
||||||
// Setup: Create a temporary directory for the namespace
|
// Setup: Create a temporary directory for the namespace
|
||||||
@@ -651,6 +761,58 @@ mod tests {
|
|||||||
assert_eq!(id_col.value(2), 30);
|
assert_eq!(id_col.value(2), 30);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_namespace_create_table_after_declare_conflict() {
|
||||||
|
let tmp_dir = tempdir().unwrap();
|
||||||
|
let root_path = tmp_dir.path().to_str().unwrap().to_string();
|
||||||
|
|
||||||
|
let mut properties = HashMap::new();
|
||||||
|
properties.insert("root".to_string(), root_path);
|
||||||
|
|
||||||
|
let conn = connect_namespace("dir", properties)
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.expect("Failed to connect to namespace");
|
||||||
|
|
||||||
|
conn.create_namespace(CreateNamespaceRequest {
|
||||||
|
id: Some(vec!["test_ns".into()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("Failed to create namespace");
|
||||||
|
|
||||||
|
let namespace_client = conn.namespace_client().await.unwrap();
|
||||||
|
namespace_client
|
||||||
|
.declare_table(DeclareTableRequest {
|
||||||
|
id: Some(vec!["test_ns".into(), "declared_test".into()]),
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.expect("Failed to declare table");
|
||||||
|
|
||||||
|
let test_data = create_test_data();
|
||||||
|
let table = conn
|
||||||
|
.create_table("declared_test", test_data)
|
||||||
|
.namespace(vec!["test_ns".into()])
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.expect("Failed to create table after declare conflict");
|
||||||
|
|
||||||
|
let results = table
|
||||||
|
.query()
|
||||||
|
.execute()
|
||||||
|
.await
|
||||||
|
.expect("Failed to query table")
|
||||||
|
.try_collect::<Vec<_>>()
|
||||||
|
.await
|
||||||
|
.expect("Failed to collect results");
|
||||||
|
|
||||||
|
assert_eq!(results.len(), 1);
|
||||||
|
assert_eq!(results[0].num_rows(), 5);
|
||||||
|
assert_eq!(table.namespace(), &["test_ns"]);
|
||||||
|
assert_eq!(table.id(), "test_ns$declared_test");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_namespace_create_table_exist_ok_mode() {
|
async fn test_namespace_create_table_exist_ok_mode() {
|
||||||
// Setup: Create a temporary directory for the namespace
|
// Setup: Create a temporary directory for the namespace
|
||||||
|
|||||||
@@ -69,7 +69,7 @@
|
|||||||
//! It treats [`FixedSizeList<Float16/Float32>`](https://docs.rs/arrow/latest/arrow/array/struct.FixedSizeListArray.html)
|
//! It treats [`FixedSizeList<Float16/Float32>`](https://docs.rs/arrow/latest/arrow/array/struct.FixedSizeListArray.html)
|
||||||
//! columns as vector columns.
|
//! columns as vector columns.
|
||||||
//!
|
//!
|
||||||
//! For more details, please refer to the [LanceDB documentation](https://lancedb.com/docs).
|
//! For more details, please refer to the [LanceDB documentation](https://docs.lancedb.com).
|
||||||
//!
|
//!
|
||||||
//! #### Create a table
|
//! #### Create a table
|
||||||
//!
|
//!
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ pub struct RemoteDatabaseOptions {
|
|||||||
pub host_override: Option<String>,
|
pub host_override: Option<String>,
|
||||||
/// Storage options configure the storage layer (e.g. S3, GCS, Azure, etc.)
|
/// Storage options configure the storage layer (e.g. S3, GCS, Azure, etc.)
|
||||||
///
|
///
|
||||||
/// See available options at <https://lancedb.com/docs/storage/>
|
/// See available options at <https://docs.lancedb.com/storage/>
|
||||||
///
|
///
|
||||||
/// These options are only used for LanceDB Enterprise and only a subset of options
|
/// These options are only used for LanceDB Enterprise and only a subset of options
|
||||||
/// are supported.
|
/// are supported.
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ use std::format;
|
|||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use crate::connection::PushdownOperation;
|
use crate::connection::NamespaceClientPushdownOperation;
|
||||||
|
|
||||||
use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
|
use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
|
||||||
use crate::database::Database;
|
use crate::database::Database;
|
||||||
@@ -1272,7 +1272,7 @@ pub struct NativeTable {
|
|||||||
pub(crate) namespace_client: Option<Arc<dyn LanceNamespace>>,
|
pub(crate) namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||||
// Operations to push down to the namespace server.
|
// Operations to push down to the namespace server.
|
||||||
// pub(crate) so query.rs can access the field for server-side query execution.
|
// pub(crate) so query.rs can access the field for server-side query execution.
|
||||||
pub(crate) pushdown_operations: HashSet<PushdownOperation>,
|
pub(crate) pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for NativeTable {
|
impl std::fmt::Debug for NativeTable {
|
||||||
@@ -1359,7 +1359,7 @@ impl NativeTable {
|
|||||||
params: Option<ReadParams>,
|
params: Option<ReadParams>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
managed_versioning: Option<bool>,
|
managed_versioning: Option<bool>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let params = params.unwrap_or_default();
|
let params = params.unwrap_or_default();
|
||||||
@@ -1470,7 +1470,7 @@ impl NativeTable {
|
|||||||
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||||
params: Option<ReadParams>,
|
params: Option<ReadParams>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
session: Option<Arc<lance::session::Session>>,
|
session: Option<Arc<lance::session::Session>>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let mut params = params.unwrap_or_default();
|
let mut params = params.unwrap_or_default();
|
||||||
@@ -1518,7 +1518,7 @@ impl NativeTable {
|
|||||||
let id = Self::build_id(&namespace, name);
|
let id = Self::build_id(&namespace, name);
|
||||||
|
|
||||||
let stored_namespace_client =
|
let stored_namespace_client =
|
||||||
if pushdown_operations.contains(&PushdownOperation::QueryTable) {
|
if pushdown_operations.contains(&NamespaceClientPushdownOperation::QueryTable) {
|
||||||
Some(namespace_client)
|
Some(namespace_client)
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@@ -1588,7 +1588,7 @@ impl NativeTable {
|
|||||||
params: Option<WriteParams>,
|
params: Option<WriteParams>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
// Default params uses format v1.
|
// Default params uses format v1.
|
||||||
let params = params.unwrap_or(WriteParams {
|
let params = params.unwrap_or(WriteParams {
|
||||||
@@ -1635,7 +1635,7 @@ impl NativeTable {
|
|||||||
params: Option<WriteParams>,
|
params: Option<WriteParams>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
let data: Box<dyn Scannable> = Box::new(RecordBatch::new_empty(schema));
|
let data: Box<dyn Scannable> = Box::new(RecordBatch::new_empty(schema));
|
||||||
Self::create(
|
Self::create(
|
||||||
@@ -1685,7 +1685,7 @@ impl NativeTable {
|
|||||||
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
|
||||||
params: Option<WriteParams>,
|
params: Option<WriteParams>,
|
||||||
read_consistency_interval: Option<std::time::Duration>,
|
read_consistency_interval: Option<std::time::Duration>,
|
||||||
pushdown_operations: HashSet<PushdownOperation>,
|
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
|
||||||
session: Option<Arc<lance::session::Session>>,
|
session: Option<Arc<lance::session::Session>>,
|
||||||
) -> Result<Self> {
|
) -> Result<Self> {
|
||||||
// Build table_id from namespace + name for the storage options provider
|
// Build table_id from namespace + name for the storage options provider
|
||||||
@@ -1738,7 +1738,7 @@ impl NativeTable {
|
|||||||
let id = Self::build_id(&namespace, name);
|
let id = Self::build_id(&namespace, name);
|
||||||
|
|
||||||
let stored_namespace_client =
|
let stored_namespace_client =
|
||||||
if pushdown_operations.contains(&PushdownOperation::QueryTable) {
|
if pushdown_operations.contains(&NamespaceClientPushdownOperation::QueryTable) {
|
||||||
Some(namespace_client)
|
Some(namespace_client)
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
|
|||||||
@@ -285,7 +285,10 @@ mod tests {
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
use crate::{connect, io::object_store::io_tracking::IoStatsHolder, table::WriteOptions};
|
use crate::{
|
||||||
|
connect, io::object_store::io_tracking::IoStatsHolder, table::WriteOptions,
|
||||||
|
utils::background_cache::clock,
|
||||||
|
};
|
||||||
|
|
||||||
async fn create_test_dataset(uri: &str) -> Dataset {
|
async fn create_test_dataset(uri: &str) -> Dataset {
|
||||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||||
@@ -463,6 +466,10 @@ mod tests {
|
|||||||
let uri = dir.path().to_str().unwrap();
|
let uri = dir.path().to_str().unwrap();
|
||||||
let ds = create_test_dataset(uri).await;
|
let ds = create_test_dataset(uri).await;
|
||||||
|
|
||||||
|
// Other tests use a thread-local mock clock. Simulate leaked state from a
|
||||||
|
// previous test to ensure this wrapper starts from real time.
|
||||||
|
clock::advance_by(Duration::from_secs(60));
|
||||||
|
|
||||||
let wrapper = DatasetConsistencyWrapper::new_latest(ds, Some(Duration::from_millis(200)));
|
let wrapper = DatasetConsistencyWrapper::new_latest(ds, Some(Duration::from_millis(200)));
|
||||||
|
|
||||||
// Populate the cache
|
// Populate the cache
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use super::NativeTable;
|
use super::NativeTable;
|
||||||
use crate::connection::PushdownOperation;
|
use crate::connection::NamespaceClientPushdownOperation;
|
||||||
use crate::error::{Error, Result};
|
use crate::error::{Error, Result};
|
||||||
use crate::expr::expr_to_sql_string;
|
use crate::expr::expr_to_sql_string;
|
||||||
use crate::query::{
|
use crate::query::{
|
||||||
@@ -44,7 +44,7 @@ pub async fn execute_query(
|
|||||||
// If QueryTable pushdown is enabled and namespace client is configured, use server-side query execution
|
// If QueryTable pushdown is enabled and namespace client is configured, use server-side query execution
|
||||||
if table
|
if table
|
||||||
.pushdown_operations
|
.pushdown_operations
|
||||||
.contains(&PushdownOperation::QueryTable)
|
.contains(&NamespaceClientPushdownOperation::QueryTable)
|
||||||
&& let Some(ref namespace_client) = table.namespace_client
|
&& let Some(ref namespace_client) = table.namespace_client
|
||||||
{
|
{
|
||||||
return execute_namespace_query(table, namespace_client.clone(), query, options).await;
|
return execute_namespace_query(table, namespace_client.clone(), query, options).await;
|
||||||
|
|||||||
@@ -107,6 +107,14 @@ where
|
|||||||
refresh_window < ttl,
|
refresh_window < ttl,
|
||||||
"refresh_window ({refresh_window:?}) must be less than ttl ({ttl:?})"
|
"refresh_window ({refresh_window:?}) must be less than ttl ({ttl:?})"
|
||||||
);
|
);
|
||||||
|
#[cfg(test)]
|
||||||
|
{
|
||||||
|
// Tests may advance the thread-local mock clock and leave it behind for
|
||||||
|
// the next test that happens to run on the same worker thread. Each new
|
||||||
|
// cache should start from a clean clock state instead of inheriting
|
||||||
|
// unrelated mock time from a previous test.
|
||||||
|
clock::clear_mock();
|
||||||
|
}
|
||||||
Self {
|
Self {
|
||||||
inner: Arc::new(Mutex::new(CacheInner {
|
inner: Arc::new(Mutex::new(CacheInner {
|
||||||
state: State::Empty,
|
state: State::Empty,
|
||||||
|
|||||||
Reference in New Issue
Block a user