Compare commits

..

14 Commits

Author SHA1 Message Date
Jack Ye f54f5600ad refactor(python): use namespace-backed rust connection 2026-04-18 17:09:13 -07:00
Jack Ye e34fe84c7f refactor(rust): align namespace client naming 2026-04-18 00:49:04 -07:00
Jack Ye 5b1f248257 fix(python): use rust namespace connection directly 2026-04-18 00:28:23 -07:00
Jack Ye 95e34d47b9 refactor(python): decouple namespace tables from temp connections 2026-04-17 20:30:52 -07:00
Jack Ye a0defd448f fix(python): handle namespace tables via resolved locations 2026-04-17 15:39:43 -07:00
Jack Ye 0fadb65153 refactor(python): document temporary namespace connections 2026-04-17 14:44:39 -07:00
Jack Ye 15fbcf61fc fix(python): use declared location for namespace table open 2026-04-17 09:58:00 -07:00
Lance Release d715bbb588 Bump version: 0.28.0-beta.6 → 0.28.0-beta.7 2026-04-17 08:12:27 +00:00
Lance Release 5ce3d8d141 Bump version: 0.31.0-beta.6 → 0.31.0-beta.7 2026-04-17 08:12:03 +00:00
Jack Ye 5eaac178b1 fix(python): pass namespace client on schema-only table create (#3283)
## Summary
- pass `namespace_client` through the Python create-table path
- ensure schema-only namespace table creation uses the namespace-aware
empty-table flow
- fix reopening namespace tables created without initial data
2026-04-17 01:11:18 -07:00
Lance Release 11af763fcd Bump version: 0.28.0-beta.5 → 0.28.0-beta.6 2026-04-16 18:57:28 +00:00
Lance Release 2ed5452e1c Bump version: 0.31.0-beta.5 → 0.31.0-beta.6 2026-04-16 18:57:05 +00:00
Xuanwo b7c0b5987c chore: upgrade lance to 6.0.0-beta.1 (#3281) 2026-04-17 02:51:58 +08:00
Jack Ye 97a4b38f19 feat(rust): support nested namespace ops in listing db (#3279)
## Summary
- delegate child-namespace `ListingDatabase` operations through an
eagerly initialized `LanceNamespaceDatabase`
- support nested namespace create/open/list/drop flows without requiring
callers to inject explicit locations
- add `namespace_client_properties` plumbing for local and namespace
connections so directory namespace settings like
`table_version_tracking_enabled` can be configured
- add regression tests for nested namespace ops and namespace client
property propagation
2026-04-16 10:12:28 -07:00
33 changed files with 447 additions and 753 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion] [tool.bumpversion]
current_version = "0.28.0-beta.5" current_version = "0.28.0-beta.7"
parse = """(?x) parse = """(?x)
(?P<major>0|[1-9]\\d*)\\. (?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\. (?P<minor>0|[1-9]\\d*)\\.
Generated
+61 -325
View File
@@ -137,15 +137,6 @@ dependencies = [
"object", "object",
] ]
[[package]]
name = "arc-swap"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6"
dependencies = [
"rustversion",
]
[[package]] [[package]]
name = "argminmax" name = "argminmax"
version = "0.6.3" version = "0.6.3"
@@ -302,7 +293,7 @@ dependencies = [
"arrow-schema", "arrow-schema",
"arrow-select", "arrow-select",
"flatbuffers", "flatbuffers",
"lz4_flex 0.12.1", "lz4_flex",
"zstd", "zstd",
] ]
@@ -1278,31 +1269,6 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "bon"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
dependencies = [
"bon-macros",
"rustversion",
]
[[package]]
name = "bon-macros"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
dependencies = [
"darling 0.20.11",
"ident_case",
"prettyplease",
"proc-macro2",
"quote",
"rustversion",
"syn 2.0.117",
]
[[package]] [[package]]
name = "brotli" name = "brotli"
version = "3.5.0" version = "3.5.0"
@@ -1472,12 +1438,6 @@ dependencies = [
"shlex", "shlex",
] ]
[[package]]
name = "census"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0"
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
version = "1.0.4" version = "1.0.4"
@@ -1795,7 +1755,7 @@ dependencies = [
"crossterm_winapi", "crossterm_winapi",
"document-features", "document-features",
"parking_lot", "parking_lot",
"rustix 1.1.4", "rustix",
"winapi", "winapi",
] ]
@@ -2714,12 +2674,6 @@ dependencies = [
"litrs", "litrs",
] ]
[[package]]
name = "downcast-rs"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc"
[[package]] [[package]]
name = "dtor" name = "dtor"
version = "0.1.1" version = "0.1.1"
@@ -2955,12 +2909,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
[[package]]
name = "fastdivide"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471"
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "2.3.0" version = "2.3.0"
@@ -3054,16 +3002,6 @@ dependencies = [
"percent-encoding", "percent-encoding",
] ]
[[package]]
name = "fs4"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8"
dependencies = [
"rustix 0.38.44",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "fs_extra" name = "fs_extra"
version = "1.3.0" version = "1.3.0"
@@ -3072,8 +3010,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]] [[package]]
name = "fsst" name = "fsst"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"rand 0.9.2", "rand 0.9.2",
@@ -3560,12 +3498,6 @@ dependencies = [
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
[[package]]
name = "htmlescape"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
[[package]] [[package]]
name = "http" name = "http"
version = "0.2.12" version = "0.2.12"
@@ -4134,8 +4066,8 @@ dependencies = [
[[package]] [[package]]
name = "lance" name = "lance"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-arith", "arrow-arith",
@@ -4178,12 +4110,14 @@ dependencies = [
"lance-linalg", "lance-linalg",
"lance-namespace", "lance-namespace",
"lance-table", "lance-table",
"lance-tokenizer",
"log", "log",
"moka", "moka",
"object_store", "object_store",
"permutation", "permutation",
"pin-project", "pin-project",
"prost", "prost",
"prost-build",
"prost-types", "prost-types",
"rand 0.9.2", "rand 0.9.2",
"roaring", "roaring",
@@ -4191,7 +4125,6 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"snafu 0.9.0", "snafu 0.9.0",
"tantivy",
"tokio", "tokio",
"tokio-stream", "tokio-stream",
"tokio-util", "tokio-util",
@@ -4202,8 +4135,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-arrow" name = "lance-arrow"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4224,8 +4157,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-bitpacking" name = "lance-bitpacking"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrayref", "arrayref",
"paste", "paste",
@@ -4234,8 +4167,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-core" name = "lance-core"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4272,8 +4205,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-datafusion" name = "lance-datafusion"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-array", "arrow-array",
@@ -4304,8 +4237,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-datagen" name = "lance-datagen"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-array", "arrow-array",
@@ -4323,8 +4256,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-encoding" name = "lance-encoding"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-arith", "arrow-arith",
"arrow-array", "arrow-array",
@@ -4361,8 +4294,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-file" name = "lance-file"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-arith", "arrow-arith",
"arrow-array", "arrow-array",
@@ -4394,8 +4327,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-index" name = "lance-index"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-arith", "arrow-arith",
@@ -4432,6 +4365,7 @@ dependencies = [
"lance-io", "lance-io",
"lance-linalg", "lance-linalg",
"lance-table", "lance-table",
"lance-tokenizer",
"libm", "libm",
"log", "log",
"ndarray", "ndarray",
@@ -4449,7 +4383,6 @@ dependencies = [
"serde_json", "serde_json",
"smallvec", "smallvec",
"snafu 0.9.0", "snafu 0.9.0",
"tantivy",
"tempfile", "tempfile",
"tokio", "tokio",
"tracing", "tracing",
@@ -4459,8 +4392,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-io" name = "lance-io"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-arith", "arrow-arith",
@@ -4504,8 +4437,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-linalg" name = "lance-linalg"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4521,8 +4454,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-namespace" name = "lance-namespace"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"async-trait", "async-trait",
@@ -4535,8 +4468,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-namespace-impls" name = "lance-namespace-impls"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-ipc", "arrow-ipc",
@@ -4581,8 +4514,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-table" name = "lance-table"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow", "arrow",
"arrow-array", "arrow-array",
@@ -4621,8 +4554,8 @@ dependencies = [
[[package]] [[package]]
name = "lance-testing" name = "lance-testing"
version = "5.1.0-beta.3" version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v5.1.0-beta.3#0a0d53b5ef4b0f65bb775ce63bdcabc79d1d14b8" source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-schema", "arrow-schema",
@@ -4631,9 +4564,19 @@ dependencies = [
"rand 0.9.2", "rand 0.9.2",
] ]
[[package]]
name = "lance-tokenizer"
version = "6.0.0-beta.1"
source = "git+https://github.com/lance-format/lance.git?tag=v6.0.0-beta.1#c7a7d3a0e944646e793d297d4a2e2cf7e4fb28a3"
dependencies = [
"rust-stemmers",
"serde",
"unicode-normalization",
]
[[package]] [[package]]
name = "lancedb" name = "lancedb"
version = "0.28.0-beta.5" version = "0.28.0-beta.7"
dependencies = [ dependencies = [
"ahash", "ahash",
"anyhow", "anyhow",
@@ -4715,7 +4658,7 @@ dependencies = [
[[package]] [[package]]
name = "lancedb-nodejs" name = "lancedb-nodejs"
version = "0.28.0-beta.5" version = "0.28.0-beta.7"
dependencies = [ dependencies = [
"arrow-array", "arrow-array",
"arrow-buffer", "arrow-buffer",
@@ -4737,7 +4680,7 @@ dependencies = [
[[package]] [[package]]
name = "lancedb-python" name = "lancedb-python"
version = "0.31.0-beta.5" version = "0.31.0-beta.7"
dependencies = [ dependencies = [
"arrow", "arrow",
"async-trait", "async-trait",
@@ -4775,12 +4718,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "levenshtein_automata"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
[[package]] [[package]]
name = "lexical-core" name = "lexical-core"
version = "1.0.6" version = "1.0.6"
@@ -4869,12 +4806,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "linux-raw-sys"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.12.1" version = "0.12.1"
@@ -4955,12 +4886,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "lz4_flex"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a"
[[package]] [[package]]
name = "lz4_flex" name = "lz4_flex"
version = "0.12.1" version = "0.12.1"
@@ -5035,15 +4960,6 @@ dependencies = [
"digest", "digest",
] ]
[[package]]
name = "measure_time"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e"
dependencies = [
"log",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.8.0" version = "2.8.0"
@@ -5197,12 +5113,6 @@ dependencies = [
"target-features", "target-features",
] ]
[[package]]
name = "murmurhash32"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b"
[[package]] [[package]]
name = "napi" name = "napi"
version = "3.8.3" version = "3.8.3"
@@ -5491,12 +5401,6 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "oneshot"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"
[[package]] [[package]]
name = "onig" name = "onig"
version = "6.5.1" version = "6.5.1"
@@ -5586,15 +5490,6 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
[[package]]
name = "ownedbytes"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e"
dependencies = [
"stable_deref_trait",
]
[[package]] [[package]]
name = "p256" name = "p256"
version = "0.11.1" version = "0.11.1"
@@ -7049,19 +6944,6 @@ dependencies = [
"semver", "semver",
] ]
[[package]]
name = "rustix"
version = "0.38.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys 0.4.15",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "rustix" name = "rustix"
version = "1.1.4" version = "1.1.4"
@@ -7071,7 +6953,7 @@ dependencies = [
"bitflags", "bitflags",
"errno", "errno",
"libc", "libc",
"linux-raw-sys 0.12.1", "linux-raw-sys",
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
@@ -7536,15 +7418,6 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
[[package]]
name = "sketches-ddsketch"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.12" version = "0.4.12"
@@ -7917,152 +7790,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
[[package]]
name = "tantivy"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43"
dependencies = [
"aho-corasick",
"arc-swap",
"base64 0.22.1",
"bitpacking",
"bon",
"byteorder",
"census",
"crc32fast",
"crossbeam-channel",
"downcast-rs",
"fastdivide",
"fnv",
"fs4",
"htmlescape",
"hyperloglogplus",
"itertools 0.14.0",
"levenshtein_automata",
"log",
"lru",
"lz4_flex 0.11.6",
"measure_time",
"memmap2 0.9.10",
"once_cell",
"oneshot",
"rayon",
"regex",
"rust-stemmers",
"rustc-hash",
"serde",
"serde_json",
"sketches-ddsketch",
"smallvec",
"tantivy-bitpacker",
"tantivy-columnar",
"tantivy-common",
"tantivy-fst",
"tantivy-query-grammar",
"tantivy-stacker",
"tantivy-tokenizer-api",
"tempfile",
"thiserror 2.0.18",
"time",
"uuid",
"winapi",
]
[[package]]
name = "tantivy-bitpacker"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494"
dependencies = [
"bitpacking",
]
[[package]]
name = "tantivy-columnar"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344"
dependencies = [
"downcast-rs",
"fastdivide",
"itertools 0.14.0",
"serde",
"tantivy-bitpacker",
"tantivy-common",
"tantivy-sstable",
"tantivy-stacker",
]
[[package]]
name = "tantivy-common"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f"
dependencies = [
"async-trait",
"byteorder",
"ownedbytes",
"serde",
"time",
]
[[package]]
name = "tantivy-fst"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18"
dependencies = [
"byteorder",
"regex-syntax",
"utf8-ranges",
]
[[package]]
name = "tantivy-query-grammar"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a"
dependencies = [
"nom 7.1.3",
"serde",
"serde_json",
]
[[package]]
name = "tantivy-sstable"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416"
dependencies = [
"futures-util",
"itertools 0.14.0",
"tantivy-bitpacker",
"tantivy-common",
"tantivy-fst",
"zstd",
]
[[package]]
name = "tantivy-stacker"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1"
dependencies = [
"murmurhash32",
"rand_distr 0.4.3",
"tantivy-common",
]
[[package]]
name = "tantivy-tokenizer-api"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "tap" name = "tap"
version = "1.0.1" version = "1.0.1"
@@ -8090,7 +7817,7 @@ dependencies = [
"fastrand", "fastrand",
"getrandom 0.4.2", "getrandom 0.4.2",
"once_cell", "once_cell",
"rustix 1.1.4", "rustix",
"windows-sys 0.59.0", "windows-sys 0.59.0",
] ]
@@ -8542,6 +8269,15 @@ version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-normalization"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
dependencies = [
"tinyvec",
]
[[package]] [[package]]
name = "unicode-normalization-alignments" name = "unicode-normalization-alignments"
version = "0.1.12" version = "0.1.12"
+14 -14
View File
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0" rust-version = "1.91.0"
[workspace.dependencies] [workspace.dependencies]
lance = { "version" = "=5.1.0-beta.3", default-features = false, "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-core = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-datagen = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-file = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=5.1.0-beta.3", default-features = false, "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-io = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-index = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-linalg = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-namespace = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=5.1.0-beta.3", default-features = false, "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-namespace-impls = { "version" = "=6.0.0-beta.1", default-features = false, "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-table = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-testing = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-datafusion = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-encoding = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=5.1.0-beta.3", "tag" = "v5.1.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lance-arrow = { "version" = "=6.0.0-beta.1", "tag" = "v6.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8" ahash = "0.8"
# Note that this one does not include pyarrow # Note that this one does not include pyarrow
arrow = { version = "57.2", optional = false } arrow = { version = "57.2", optional = false }
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency> <dependency>
<groupId>com.lancedb</groupId> <groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId> <artifactId>lancedb-core</artifactId>
<version>0.28.0-beta.5</version> <version>0.28.0-beta.7</version>
</dependency> </dependency>
``` ```
+1 -1
View File
@@ -8,7 +8,7 @@
<parent> <parent>
<groupId>com.lancedb</groupId> <groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId> <artifactId>lancedb-parent</artifactId>
<version>0.28.0-beta.5</version> <version>0.28.0-beta.7</version>
<relativePath>../pom.xml</relativePath> <relativePath>../pom.xml</relativePath>
</parent> </parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId> <groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId> <artifactId>lancedb-parent</artifactId>
<version>0.28.0-beta.5</version> <version>0.28.0-beta.7</version>
<packaging>pom</packaging> <packaging>pom</packaging>
<name>${project.artifactId}</name> <name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description> <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties> <properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version> <arrow.version>15.0.0</arrow.version>
<lance-core.version>5.1.0-beta.3</lance-core.version> <lance-core.version>6.0.0-beta.1</lance-core.version>
<spotless.skip>false</spotless.skip> <spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version> <spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version> <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package] [package]
name = "lancedb-nodejs" name = "lancedb-nodejs"
edition.workspace = true edition.workspace = true
version = "0.28.0-beta.5" version = "0.28.0-beta.7"
license.workspace = true license.workspace = true
description.workspace = true description.workspace = true
repository.workspace = true repository.workspace = true
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-darwin-arm64", "name": "@lancedb/lancedb-darwin-arm64",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": ["darwin"], "os": ["darwin"],
"cpu": ["arm64"], "cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node", "main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-arm64-gnu", "name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": ["linux"], "os": ["linux"],
"cpu": ["arm64"], "cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node", "main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-arm64-musl", "name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": ["linux"], "os": ["linux"],
"cpu": ["arm64"], "cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node", "main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-x64-gnu", "name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": ["linux"], "os": ["linux"],
"cpu": ["x64"], "cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node", "main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-linux-x64-musl", "name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": ["linux"], "os": ["linux"],
"cpu": ["x64"], "cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node", "main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-win32-arm64-msvc", "name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": [ "os": [
"win32" "win32"
], ],
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@lancedb/lancedb-win32-x64-msvc", "name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"os": ["win32"], "os": ["win32"],
"cpu": ["x64"], "cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node", "main": "lancedb.win32-x64-msvc.node",
+2 -2
View File
@@ -1,12 +1,12 @@
{ {
"name": "@lancedb/lancedb", "name": "@lancedb/lancedb",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@lancedb/lancedb", "name": "@lancedb/lancedb",
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"cpu": [ "cpu": [
"x64", "x64",
"arm64" "arm64"
+1 -1
View File
@@ -11,7 +11,7 @@
"ann" "ann"
], ],
"private": false, "private": false,
"version": "0.28.0-beta.5", "version": "0.28.0-beta.7",
"main": "dist/index.js", "main": "dist/index.js",
"exports": { "exports": {
".": "./dist/index.js", ".": "./dist/index.js",
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion] [tool.bumpversion]
current_version = "0.31.0-beta.5" current_version = "0.31.0-beta.7"
parse = """(?x) parse = """(?x)
(?P<major>0|[1-9]\\d*)\\. (?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\. (?P<minor>0|[1-9]\\d*)\\.
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "lancedb-python" name = "lancedb-python"
version = "0.31.0-beta.5" version = "0.31.0-beta.7"
edition.workspace = true edition.workspace = true
description = "Python bindings for LanceDB" description = "Python bindings for LanceDB"
license.workspace = true license.workspace = true
+3
View File
@@ -1390,6 +1390,7 @@ class AsyncConnection(object):
namespace_path: Optional[List[str]] = None, namespace_path: Optional[List[str]] = None,
embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None, embedding_functions: Optional[List[EmbeddingFunctionConfig]] = None,
location: Optional[str] = None, location: Optional[str] = None,
namespace_client: Optional[Any] = None,
) -> AsyncTable: ) -> AsyncTable:
"""Create an [AsyncTable][lancedb.table.AsyncTable] in the database. """Create an [AsyncTable][lancedb.table.AsyncTable] in the database.
@@ -1587,6 +1588,7 @@ class AsyncConnection(object):
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=storage_options, storage_options=storage_options,
location=location, location=location,
namespace_client=namespace_client,
) )
else: else:
data = data_to_reader(data, schema) data = data_to_reader(data, schema)
@@ -1597,6 +1599,7 @@ class AsyncConnection(object):
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=storage_options, storage_options=storage_options,
location=location, location=location,
namespace_client=namespace_client,
) )
return AsyncTable(new_table) return AsyncTable(new_table)
+111 -357
View File
@@ -10,7 +10,6 @@ through a namespace abstraction.
from __future__ import annotations from __future__ import annotations
import asyncio
import sys import sys
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
@@ -25,7 +24,24 @@ if TYPE_CHECKING:
from datetime import timedelta from datetime import timedelta
import pyarrow as pa import pyarrow as pa
from lancedb.db import DBConnection, LanceDBConnection from lance_namespace_urllib3_client.models.json_arrow_data_type import JsonArrowDataType
from lance_namespace_urllib3_client.models.json_arrow_field import JsonArrowField
from lance_namespace_urllib3_client.models.json_arrow_schema import JsonArrowSchema
from lance_namespace_urllib3_client.models.query_table_request import QueryTableRequest
from lance_namespace_urllib3_client.models.query_table_request_columns import (
QueryTableRequestColumns,
)
from lance_namespace_urllib3_client.models.query_table_request_full_text_query import (
QueryTableRequestFullTextQuery,
)
from lance_namespace_urllib3_client.models.query_table_request_vector import (
QueryTableRequestVector,
)
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
from lance_namespace.errors import TableNotFoundError
from lancedb._lancedb import connect_namespace_client as _connect_namespace_client
from lancedb.background_loop import LOOP
from lancedb.db import AsyncConnection, DBConnection
from lancedb.namespace_utils import ( from lancedb.namespace_utils import (
_normalize_create_namespace_mode, _normalize_create_namespace_mode,
_normalize_drop_namespace_mode, _normalize_drop_namespace_mode,
@@ -40,14 +56,11 @@ from lance_namespace import (
ListNamespacesResponse, ListNamespacesResponse,
ListTablesResponse, ListTablesResponse,
ListTablesRequest, ListTablesRequest,
DescribeTableRequest,
DescribeNamespaceRequest, DescribeNamespaceRequest,
DropTableRequest, DropTableRequest,
ListNamespacesRequest, ListNamespacesRequest,
CreateNamespaceRequest, CreateNamespaceRequest,
DropNamespaceRequest, DropNamespaceRequest,
DeclareTableRequest,
CreateTableRequest,
) )
from lancedb.table import AsyncTable, LanceTable, Table from lancedb.table import AsyncTable, LanceTable, Table
from lancedb.util import validate_table_name from lancedb.util import validate_table_name
@@ -56,21 +69,6 @@ from lancedb.pydantic import LanceModel
from lancedb.embeddings import EmbeddingFunctionConfig from lancedb.embeddings import EmbeddingFunctionConfig
from ._lancedb import Session from ._lancedb import Session
from lance_namespace_urllib3_client.models.json_arrow_schema import JsonArrowSchema
from lance_namespace_urllib3_client.models.json_arrow_field import JsonArrowField
from lance_namespace_urllib3_client.models.json_arrow_data_type import JsonArrowDataType
from lance_namespace_urllib3_client.models.query_table_request import QueryTableRequest
from lance_namespace_urllib3_client.models.query_table_request_vector import (
QueryTableRequestVector,
)
from lance_namespace_urllib3_client.models.query_table_request_columns import (
QueryTableRequestColumns,
)
from lance_namespace_urllib3_client.models.query_table_request_full_text_query import (
QueryTableRequestFullTextQuery,
)
from lance_namespace_urllib3_client.models.string_fts_query import StringFtsQuery
def _query_to_namespace_request( def _query_to_namespace_request(
table_id: List[str], table_id: List[str],
@@ -424,6 +422,23 @@ class LanceNamespaceDBConnection(DBConnection):
) )
self._namespace_client_impl = namespace_client_impl self._namespace_client_impl = namespace_client_impl
self._namespace_client_properties = namespace_client_properties self._namespace_client_properties = namespace_client_properties
self._inner = AsyncConnection(
_connect_namespace_client(
namespace_client,
read_consistency_interval=(
read_consistency_interval.total_seconds()
if read_consistency_interval is not None
else None
),
storage_options=self.storage_options or None,
session=session,
namespace_client_pushdown_operations=(
list(self._namespace_client_pushdown_operations)
),
namespace_client_impl=namespace_client_impl,
namespace_client_properties=namespace_client_properties,
)
)
@override @override
def serialize(self) -> str: def serialize(self) -> str:
@@ -497,13 +512,10 @@ class LanceNamespaceDBConnection(DBConnection):
if mode.lower() not in ["create", "overwrite"]: if mode.lower() not in ["create", "overwrite"]:
raise ValueError("mode must be either 'create' or 'overwrite'") raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name) validate_table_name(name)
async_table = LOOP.run(
table_id = namespace_path + [name] self._inner.create_table(
name,
if "CreateTable" in self._namespace_client_pushdown_operations: data,
return self._create_table_server_side(
name=name,
data=data,
schema=schema, schema=schema,
mode=mode, mode=mode,
exist_ok=exist_ok, exist_ok=exist_ok,
@@ -513,130 +525,15 @@ class LanceNamespaceDBConnection(DBConnection):
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=storage_options, storage_options=storage_options,
) )
# Local create path: declare_table + local write
# Step 1: Get the table location and storage options from namespace
# In overwrite mode, if table exists, use describe_table to get
# existing location. Otherwise, call create_empty_table to reserve
# a new location
location = None
namespace_storage_options = None
if mode.lower() == "overwrite":
# Try to describe the table first to see if it exists
try:
describe_request = DescribeTableRequest(id=table_id)
describe_response = self._namespace_client.describe_table(
describe_request
)
location = describe_response.location
namespace_storage_options = describe_response.storage_options
except Exception:
# Table doesn't exist, will create a new one below
pass
if location is None:
# Table doesn't exist or mode is "create", reserve a new location
declare_request = DeclareTableRequest(
id=table_id,
location=None,
properties=self.storage_options if self.storage_options else None,
)
declare_response = self._namespace_client.declare_table(declare_request)
if not declare_response.location:
raise ValueError(
"Table location is missing from declare_table response"
)
location = declare_response.location
namespace_storage_options = declare_response.storage_options
# Merge storage options: self.storage_options < user options < namespace options
merged_storage_options = dict(self.storage_options)
if storage_options:
merged_storage_options.update(storage_options)
if namespace_storage_options:
merged_storage_options.update(namespace_storage_options)
# Step 2: Create table using LanceTable.create with the location
# We need a temporary connection for the LanceTable.create method
temp_conn = LanceDBConnection(
location, # Use the actual location as the connection URI
read_consistency_interval=self.read_consistency_interval,
storage_options=merged_storage_options,
session=self.session,
) )
# Note: storage_options_provider is auto-created in Rust from namespace_client return LanceTable(
tbl = LanceTable.create( self,
temp_conn,
name, name,
data,
schema,
mode=mode,
exist_ok=exist_ok,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=merged_storage_options,
location=location,
namespace_client=self._namespace_client, namespace_client=self._namespace_client,
pushdown_operations=self._namespace_client_pushdown_operations, pushdown_operations=self._namespace_client_pushdown_operations,
) _async=async_table,
return tbl
def _create_table_server_side(
self,
name: str,
data: Optional[DATA],
schema: Optional[Union[pa.Schema, LanceModel]],
mode: str,
exist_ok: bool,
on_bad_vectors: str,
fill_value: float,
embedding_functions: Optional[List[EmbeddingFunctionConfig]],
namespace_path: Optional[List[str]],
storage_options: Optional[Dict[str, str]],
) -> Table:
"""Create a table using server-side namespace.create_table()."""
if namespace_path is None:
namespace_path = []
table_id = namespace_path + [name]
arrow_ipc_bytes = _data_to_arrow_ipc(
data=data,
schema=schema,
embedding_functions=embedding_functions,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
)
merged = dict(self.storage_options or {})
if storage_options:
merged.update(storage_options)
request = CreateTableRequest(
id=table_id,
mode=_normalize_create_table_mode(mode),
properties=merged or None,
)
try:
self._namespace_client.create_table(request, arrow_ipc_bytes)
except Exception as e:
if exist_ok and "already exists" in str(e).lower():
return self.open_table(
name,
namespace_path=namespace_path,
storage_options=storage_options,
)
raise
return self.open_table(
name,
namespace_path=namespace_path,
storage_options=storage_options,
) )
@override @override
@@ -650,30 +547,28 @@ class LanceNamespaceDBConnection(DBConnection):
) -> Table: ) -> Table:
if namespace_path is None: if namespace_path is None:
namespace_path = [] namespace_path = []
table_id = namespace_path + [name] try:
request = DescribeTableRequest(id=table_id) async_table = LOOP.run(
response = self._namespace_client.describe_table(request) self._inner.open_table(
name,
namespace_path=namespace_path,
storage_options=storage_options,
index_cache_size=index_cache_size,
)
)
except RuntimeError as e:
if "Table not found" in str(e):
table_id = namespace_path + [name]
raise TableNotFoundError(f"Table not found: {'$'.join(table_id)}")
raise
# Merge storage options: self.storage_options < user options < namespace options return LanceTable(
merged_storage_options = dict(self.storage_options) self,
if storage_options:
merged_storage_options.update(storage_options)
if response.storage_options:
merged_storage_options.update(response.storage_options)
# Pass managed_versioning to avoid redundant describe_table call in Rust.
# Convert None to False since we already have the answer from describe_table.
managed_versioning = response.managed_versioning is True
# Note: storage_options_provider is auto-created in Rust from namespace_client
return self._lance_table_from_uri(
name, name,
response.location,
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=merged_storage_options,
index_cache_size=index_cache_size,
namespace_client=self._namespace_client, namespace_client=self._namespace_client,
managed_versioning=managed_versioning, pushdown_operations=self._namespace_client_pushdown_operations,
_async=async_table,
) )
@override @override
@@ -897,33 +792,34 @@ class LanceNamespaceDBConnection(DBConnection):
namespace_client: Optional[Any] = None, namespace_client: Optional[Any] = None,
managed_versioning: Optional[bool] = None, managed_versioning: Optional[bool] = None,
) -> LanceTable: ) -> LanceTable:
# Open a table directly from a URI using the location parameter # Open a table directly from the namespace-resolved physical location.
# Note: storage_options should already be merged by the caller #
# Note: storage_options_provider is auto-created in Rust from namespace_client # Open the table through the Rust namespace-backed connection. The Rust
# layer keeps the logical namespace path and namespace client intact.
if namespace_path is None: if namespace_path is None:
namespace_path = [] namespace_path = []
temp_conn = LanceDBConnection(
table_uri, # Use the table location as the connection URI async_table = LOOP.run(
read_consistency_interval=self.read_consistency_interval, self._inner.open_table(
storage_options=storage_options if storage_options is not None else {}, name,
session=self.session, namespace_path=namespace_path,
storage_options=storage_options,
index_cache_size=index_cache_size,
location=None,
namespace_client=namespace_client,
managed_versioning=managed_versioning,
)
) )
# Open the table using the temporary connection with the location parameter return LanceTable(
# Pass namespace_client to enable managed versioning support and auto-create self,
# storage options provider
# Pass managed_versioning to avoid redundant describe_table call
# Pass pushdown_operations if configured on this connection
return LanceTable.open(
temp_conn,
name, name,
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=storage_options,
index_cache_size=index_cache_size,
location=table_uri, location=table_uri,
namespace_client=namespace_client, namespace_client=namespace_client,
managed_versioning=managed_versioning, managed_versioning=managed_versioning,
pushdown_operations=self._namespace_client_pushdown_operations, pushdown_operations=self._namespace_client_pushdown_operations,
_async=async_table,
) )
@override @override
@@ -990,6 +886,23 @@ class AsyncLanceNamespaceDBConnection:
self._namespace_client_pushdown_operations = set( self._namespace_client_pushdown_operations = set(
namespace_client_pushdown_operations or [] namespace_client_pushdown_operations or []
) )
self._inner = AsyncConnection(
_connect_namespace_client(
namespace_client,
read_consistency_interval=(
read_consistency_interval.total_seconds()
if read_consistency_interval is not None
else None
),
storage_options=self.storage_options or None,
session=session,
namespace_client_pushdown_operations=(
list(self._namespace_client_pushdown_operations)
),
namespace_client_impl=None,
namespace_client_properties=None,
)
)
async def table_names( async def table_names(
self, self,
@@ -1041,148 +954,16 @@ class AsyncLanceNamespaceDBConnection:
if mode.lower() not in ["create", "overwrite"]: if mode.lower() not in ["create", "overwrite"]:
raise ValueError("mode must be either 'create' or 'overwrite'") raise ValueError("mode must be either 'create' or 'overwrite'")
validate_table_name(name) validate_table_name(name)
return await self._inner.create_table(
table_id = namespace_path + [name]
if "CreateTable" in self._namespace_client_pushdown_operations:
return await self._create_table_server_side(
name=name,
data=data,
schema=schema,
mode=mode,
exist_ok=exist_ok,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
namespace_path=namespace_path,
storage_options=storage_options,
)
# Local create path: declare_table + local write
# Step 1: Get the table location and storage options from namespace
location = None
namespace_storage_options = None
if mode.lower() == "overwrite":
# Try to describe the table first to see if it exists
try:
describe_request = DescribeTableRequest(id=table_id)
describe_response = self._namespace_client.describe_table(
describe_request
)
location = describe_response.location
namespace_storage_options = describe_response.storage_options
except Exception:
# Table doesn't exist, will create a new one below
pass
if location is None:
# Table doesn't exist or mode is "create", reserve a new location
declare_request = DeclareTableRequest(
id=table_id,
location=None,
properties=self.storage_options if self.storage_options else None,
)
declare_response = self._namespace_client.declare_table(declare_request)
if not declare_response.location:
raise ValueError(
"Table location is missing from declare_table response"
)
location = declare_response.location
namespace_storage_options = declare_response.storage_options
# Merge storage options: self.storage_options < user options < namespace options
merged_storage_options = dict(self.storage_options)
if storage_options:
merged_storage_options.update(storage_options)
if namespace_storage_options:
merged_storage_options.update(namespace_storage_options)
# Step 2: Create table using LanceTable.create with the location
# Run the sync operation in a thread
def _create_table():
temp_conn = LanceDBConnection(
location,
read_consistency_interval=self.read_consistency_interval,
storage_options=merged_storage_options,
session=self.session,
)
# storage_options_provider is auto-created in Rust from namespace_client
return LanceTable.create(
temp_conn,
name,
data,
schema,
mode=mode,
exist_ok=exist_ok,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
namespace_path=namespace_path,
storage_options=merged_storage_options,
location=location,
namespace_client=self._namespace_client,
pushdown_operations=self._namespace_client_pushdown_operations,
)
lance_table = await asyncio.to_thread(_create_table)
# Get the underlying async table from LanceTable
return lance_table._table
async def _create_table_server_side(
self,
name: str,
data: Optional[DATA],
schema: Optional[Union[pa.Schema, LanceModel]],
mode: str,
exist_ok: bool,
on_bad_vectors: str,
fill_value: float,
embedding_functions: Optional[List[EmbeddingFunctionConfig]],
namespace_path: Optional[List[str]],
storage_options: Optional[Dict[str, str]],
) -> AsyncTable:
"""Create a table using server-side namespace.create_table()."""
if namespace_path is None:
namespace_path = []
table_id = namespace_path + [name]
def _prepare_and_create():
arrow_ipc_bytes = _data_to_arrow_ipc(
data=data,
schema=schema,
embedding_functions=embedding_functions,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
)
merged = dict(self.storage_options or {})
if storage_options:
merged.update(storage_options)
request = CreateTableRequest(
id=table_id,
mode=_normalize_create_table_mode(mode),
properties=merged or None,
)
self._namespace_client.create_table(request, arrow_ipc_bytes)
try:
await asyncio.to_thread(_prepare_and_create)
except Exception as e:
if exist_ok and "already exists" in str(e).lower():
return await self.open_table(
name,
namespace_path=namespace_path,
storage_options=storage_options,
)
raise
return await self.open_table(
name, name,
data,
schema=schema,
mode=mode,
exist_ok=exist_ok,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
namespace_path=namespace_path, namespace_path=namespace_path,
embedding_functions=embedding_functions,
storage_options=storage_options, storage_options=storage_options,
) )
@@ -1197,45 +978,18 @@ class AsyncLanceNamespaceDBConnection:
"""Open an existing table from the namespace.""" """Open an existing table from the namespace."""
if namespace_path is None: if namespace_path is None:
namespace_path = [] namespace_path = []
table_id = namespace_path + [name] try:
request = DescribeTableRequest(id=table_id) return await self._inner.open_table(
response = self._namespace_client.describe_table(request)
# Merge storage options: self.storage_options < user options < namespace options
merged_storage_options = dict(self.storage_options)
if storage_options:
merged_storage_options.update(storage_options)
if response.storage_options:
merged_storage_options.update(response.storage_options)
# Capture managed_versioning from describe response.
# Convert None to False since we already have the answer from describe_table.
managed_versioning = response.managed_versioning is True
# Open table in a thread
# Note: storage_options_provider is auto-created in Rust from namespace_client
def _open_table():
temp_conn = LanceDBConnection(
response.location,
read_consistency_interval=self.read_consistency_interval,
storage_options=merged_storage_options,
session=self.session,
)
return LanceTable.open(
temp_conn,
name, name,
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=merged_storage_options, storage_options=storage_options,
index_cache_size=index_cache_size, index_cache_size=index_cache_size,
location=response.location,
namespace_client=self._namespace_client,
managed_versioning=managed_versioning,
pushdown_operations=self._namespace_client_pushdown_operations,
) )
except RuntimeError as e:
lance_table = await asyncio.to_thread(_open_table) if "Table not found" in str(e):
return lance_table._table table_id = namespace_path + [name]
raise TableNotFoundError(f"Table not found: {'$'.join(table_id)}")
raise
async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the namespace.""" """Drop a table from the namespace."""
+1
View File
@@ -2929,6 +2929,7 @@ class LanceTable(Table):
namespace_path=namespace_path, namespace_path=namespace_path,
storage_options=storage_options, storage_options=storage_options,
location=location, location=location,
namespace_client=namespace_client,
) )
) )
return self return self
@@ -18,6 +18,9 @@ Tests verify:
""" """
import copy import copy
import shutil
import sys
import tempfile
import time import time
import uuid import uuid
from typing import Dict, Optional from typing import Dict, Optional
@@ -387,6 +390,66 @@ def test_namespace_open_table_with_provider(s3_bucket: str, use_custom: bool):
assert get_describe_call_count(inner_ns_client) == describe_count_after_open assert get_describe_call_count(inner_ns_client) == describe_count_after_open
@pytest.mark.skipif(
sys.platform == "win32",
reason="TODO: fix schema-only namespace metrics test on Windows",
)
@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
def test_namespace_create_schema_only_with_provider(use_custom: bool):
"""
Test creating a schema-only table through namespace with storage options provider.
Verifies:
- declare_table is called once to reserve the location
- describe_table is not needed during create in create mode
- the table can be reopened successfully afterward
- opening the table triggers exactly one describe_table call
"""
temp_dir = tempfile.mkdtemp()
try:
ns_client, inner_ns_client = create_tracking_namespace(
bucket_name=temp_dir,
storage_options={},
credential_expires_in_seconds=3600,
use_custom=use_custom,
)
db = LanceNamespaceDBConnection(ns_client)
namespace_name = f"test_ns_{uuid.uuid4().hex[:8]}"
db.create_namespace([namespace_name])
table_name = f"test_table_{uuid.uuid4().hex}"
namespace_path = [namespace_name]
schema = pa.schema(
[
pa.field("id", pa.int64()),
pa.field("vector", pa.list_(pa.float32(), 2)),
pa.field("text", pa.string()),
]
)
assert get_declare_call_count(inner_ns_client) == 0
assert get_describe_call_count(inner_ns_client) == 0
table = db.create_table(
table_name, schema=schema, namespace_path=namespace_path
)
assert table.name == table_name
assert table.namespace == namespace_path
assert get_declare_call_count(inner_ns_client) == 1
assert get_describe_call_count(inner_ns_client) == 0
reopened_table = db.open_table(table_name, namespace_path=namespace_path)
assert reopened_table.schema == schema
assert get_declare_call_count(inner_ns_client) == 1
assert get_describe_call_count(inner_ns_client) == 1
finally:
shutil.rmtree(temp_dir, ignore_errors=True)
@pytest.mark.s3_test @pytest.mark.s3_test
@pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"]) @pytest.mark.parametrize("use_custom", [False, True], ids=["DirectoryNS", "CustomNS"])
def test_namespace_credential_refresh_on_read(s3_bucket: str, use_custom: bool): def test_namespace_credential_refresh_on_read(s3_bucket: str, use_custom: bool):
+8 -1
View File
@@ -3,6 +3,7 @@
import os import os
import sys
from datetime import date, datetime, timedelta from datetime import date, datetime, timedelta
from time import sleep from time import sleep
from typing import List from typing import List
@@ -2040,6 +2041,13 @@ def test_hybrid_search_metric_type(tmp_db: DBConnection):
@pytest.mark.parametrize( @pytest.mark.parametrize(
"consistency_interval", [None, timedelta(seconds=0), timedelta(seconds=0.1)] "consistency_interval", [None, timedelta(seconds=0), timedelta(seconds=0.1)]
) )
@pytest.mark.skipif(
sys.platform == "win32",
reason=(
"TODO: directory namespace is not supported on Windows yet; "
"re-enable after that is fixed."
),
)
def test_consistency(tmp_path, consistency_interval): def test_consistency(tmp_path, consistency_interval):
db = lancedb.connect(tmp_path) db = lancedb.connect(tmp_path)
table = db.create_table("my_table", data=[{"id": 0}]) table = db.create_table("my_table", data=[{"id": 0}])
@@ -2060,7 +2068,6 @@ def test_consistency(tmp_path, consistency_interval):
elif consistency_interval == timedelta(seconds=0): elif consistency_interval == timedelta(seconds=0):
assert table2.version == table.version assert table2.version == table.version
else: else:
# (consistency_interval == timedelta(seconds=0.1)
assert table2.version == table.version - 1 assert table2.version == table.version - 1
sleep(0.1) sleep(0.1)
assert table2.version == table.version assert table2.version == table.version
+76 -1
View File
@@ -1,11 +1,17 @@
// SPDX-License-Identifier: Apache-2.0 // SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors // SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{collections::HashMap, sync::Arc, time::Duration}; use std::{
collections::{HashMap, HashSet},
sync::Arc,
time::Duration,
};
use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow}; use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
use lancedb::{ use lancedb::{
connection::Connection as LanceConnection, connection::Connection as LanceConnection,
connection::NamespaceClientPushdownOperation,
database::namespace::LanceNamespaceDatabase,
database::{CreateTableMode, Database, ReadConsistency}, database::{CreateTableMode, Database, ReadConsistency},
}; };
use pyo3::{ use pyo3::{
@@ -39,6 +45,29 @@ impl Connection {
} }
} }
fn parse_namespace_client_pushdown_operations(
operations: Option<Vec<String>>,
) -> PyResult<HashSet<NamespaceClientPushdownOperation>> {
let mut parsed = HashSet::new();
for operation in operations.unwrap_or_default() {
match operation.as_str() {
"QueryTable" => {
parsed.insert(NamespaceClientPushdownOperation::QueryTable);
}
"CreateTable" => {
parsed.insert(NamespaceClientPushdownOperation::CreateTable);
}
_ => {
return Err(PyValueError::new_err(format!(
"Invalid pushdown operation: {}",
operation
)));
}
}
}
Ok(parsed)
}
impl Connection { impl Connection {
fn parse_create_mode_str(mode: &str) -> PyResult<CreateTableMode> { fn parse_create_mode_str(mode: &str) -> PyResult<CreateTableMode> {
match mode { match mode {
@@ -538,6 +567,52 @@ pub fn connect(
}) })
} }
#[pyfunction]
#[pyo3(signature = (
namespace_client,
read_consistency_interval=None,
storage_options=None,
session=None,
namespace_client_pushdown_operations=None,
namespace_client_impl=None,
namespace_client_properties=None,
))]
#[allow(clippy::too_many_arguments)]
pub fn connect_namespace_client(
py: Python<'_>,
namespace_client: Py<PyAny>,
read_consistency_interval: Option<f64>,
storage_options: Option<HashMap<String, String>>,
session: Option<crate::session::Session>,
namespace_client_pushdown_operations: Option<Vec<String>>,
namespace_client_impl: Option<String>,
namespace_client_properties: Option<HashMap<String, String>>,
) -> PyResult<Connection> {
let namespace_client = extract_namespace_arc(py, namespace_client)?;
let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
let namespace_client_pushdown_operations =
parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
let ns_impl = namespace_client_impl.unwrap_or_else(|| "python".to_string());
let ns_properties = namespace_client_properties.unwrap_or_default();
let storage_options = storage_options.unwrap_or_default();
let session = session.map(|s| s.inner.clone());
let database = LanceNamespaceDatabase::from_namespace_client(
namespace_client,
ns_impl,
ns_properties,
storage_options,
read_consistency_interval,
session,
namespace_client_pushdown_operations,
);
Ok(Connection::new(LanceConnection::new(
Arc::new(database),
Arc::new(lancedb::embeddings::MemoryRegistry::new()),
)))
}
#[derive(FromPyObject)] #[derive(FromPyObject)]
pub struct PyClientConfig { pub struct PyClientConfig {
user_agent: String, user_agent: String,
+2 -1
View File
@@ -2,7 +2,7 @@
// SPDX-FileCopyrightText: Copyright The LanceDB Authors // SPDX-FileCopyrightText: Copyright The LanceDB Authors
use arrow::RecordBatchStream; use arrow::RecordBatchStream;
use connection::{Connection, connect}; use connection::{Connection, connect, connect_namespace_client};
use env_logger::Env; use env_logger::Env;
use expr::{PyExpr, expr_col, expr_func, expr_lit}; use expr::{PyExpr, expr_col, expr_func, expr_lit};
use index::IndexConfig; use index::IndexConfig;
@@ -58,6 +58,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PyPermutationReader>()?; m.add_class::<PyPermutationReader>()?;
m.add_class::<PyExpr>()?; m.add_class::<PyExpr>()?;
m.add_function(wrap_pyfunction!(connect, m)?)?; m.add_function(wrap_pyfunction!(connect, m)?)?;
m.add_function(wrap_pyfunction!(connect_namespace_client, m)?)?;
m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?; m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?;
m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?; m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?; m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?;
+1 -1
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "lancedb" name = "lancedb"
version = "0.28.0-beta.5" version = "0.28.0-beta.7"
edition.workspace = true edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications" description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true license.workspace = true
+6 -6
View File
@@ -915,7 +915,7 @@ use std::collections::HashSet;
/// These operations will be executed on the namespace server instead of locally /// These operations will be executed on the namespace server instead of locally
/// when enabled via [`ConnectNamespaceBuilder::pushdown_operations`]. /// when enabled via [`ConnectNamespaceBuilder::pushdown_operations`].
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum PushdownOperation { pub enum NamespaceClientPushdownOperation {
/// Execute queries on the namespace server via `query_table()` instead of locally. /// Execute queries on the namespace server via `query_table()` instead of locally.
QueryTable, QueryTable,
/// Execute table creation on the namespace server via `create_table()` /// Execute table creation on the namespace server via `create_table()`
@@ -931,7 +931,7 @@ pub struct ConnectNamespaceBuilder {
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
embedding_registry: Option<Arc<dyn EmbeddingRegistry>>, embedding_registry: Option<Arc<dyn EmbeddingRegistry>>,
session: Option<Arc<lance::session::Session>>, session: Option<Arc<lance::session::Session>>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
} }
impl ConnectNamespaceBuilder { impl ConnectNamespaceBuilder {
@@ -1029,11 +1029,11 @@ impl ConnectNamespaceBuilder {
/// and leveraging server-side compute resources. /// and leveraging server-side compute resources.
/// ///
/// Available operations: /// Available operations:
/// - [`PushdownOperation::QueryTable`]: Execute queries via `namespace.query_table()` /// - [`NamespaceClientPushdownOperation::QueryTable`]: Execute queries via `namespace.query_table()`
/// - [`PushdownOperation::CreateTable`]: Execute table creation via `namespace.create_table()` /// - [`NamespaceClientPushdownOperation::CreateTable`]: Execute table creation via `namespace.create_table()`
/// ///
/// By default, no operations are pushed down (all executed locally). /// By default, no operations are pushed down (all executed locally).
pub fn pushdown_operation(mut self, operation: PushdownOperation) -> Self { pub fn pushdown_operation(mut self, operation: NamespaceClientPushdownOperation) -> Self {
self.pushdown_operations.insert(operation); self.pushdown_operations.insert(operation);
self self
} }
@@ -1043,7 +1043,7 @@ impl ConnectNamespaceBuilder {
/// See [`Self::pushdown_operation`] for details. /// See [`Self::pushdown_operation`] for details.
pub fn pushdown_operations( pub fn pushdown_operations(
mut self, mut self,
operations: impl IntoIterator<Item = PushdownOperation>, operations: impl IntoIterator<Item = NamespaceClientPushdownOperation>,
) -> Self { ) -> Self {
self.pushdown_operations.extend(operations); self.pushdown_operations.extend(operations);
self self
+1
View File
@@ -2184,6 +2184,7 @@ mod tests {
} }
#[tokio::test] #[tokio::test]
#[cfg(not(windows))] // TODO: support Windows once directory namespace-backed listing DB tests are supported.
async fn test_listing_database_with_namespace_client_properties() { async fn test_listing_database_with_namespace_client_properties() {
let tempdir = tempdir().unwrap(); let tempdir = tempdir().unwrap();
let uri = tempdir.path().to_str().unwrap(); let uri = tempdir.path().to_str().unwrap();
+55 -17
View File
@@ -22,7 +22,7 @@ use lance_namespace_impls::ConnectBuilder;
use lance_table::io::commit::CommitHandler; use lance_table::io::commit::CommitHandler;
use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler; use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
use crate::connection::PushdownOperation; use crate::connection::NamespaceClientPushdownOperation;
use crate::database::ReadConsistency; use crate::database::ReadConsistency;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::table::NativeTable; use crate::table::NativeTable;
@@ -44,7 +44,7 @@ pub struct LanceNamespaceDatabase {
// database URI // database URI
uri: String, uri: String,
// Operations to push down to the namespace server // Operations to push down to the namespace server
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
// Namespace implementation type (e.g., "dir", "rest") // Namespace implementation type (e.g., "dir", "rest")
ns_impl: String, ns_impl: String,
// Namespace properties used to construct the namespace client // Namespace properties used to construct the namespace client
@@ -52,13 +52,34 @@ pub struct LanceNamespaceDatabase {
} }
impl LanceNamespaceDatabase { impl LanceNamespaceDatabase {
pub fn from_namespace_client(
namespace_client: Arc<dyn LanceNamespace>,
namespace_client_impl: String,
namespace_client_properties: HashMap<String, String>,
storage_options: HashMap<String, String>,
read_consistency_interval: Option<std::time::Duration>,
session: Option<Arc<lance::session::Session>>,
namespace_client_pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
) -> Self {
Self {
namespace: namespace_client,
storage_options,
read_consistency_interval,
session,
uri: format!("namespace://{}", namespace_client_impl),
pushdown_operations: namespace_client_pushdown_operations,
ns_impl: namespace_client_impl,
ns_properties: namespace_client_properties,
}
}
pub async fn connect( pub async fn connect(
ns_impl: &str, ns_impl: &str,
ns_properties: HashMap<String, String>, ns_properties: HashMap<String, String>,
storage_options: HashMap<String, String>, storage_options: HashMap<String, String>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
session: Option<Arc<lance::session::Session>>, session: Option<Arc<lance::session::Session>>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
) -> Result<Self> { ) -> Result<Self> {
let mut builder = ConnectBuilder::new(ns_impl); let mut builder = ConnectBuilder::new(ns_impl);
for (key, value) in ns_properties.clone() { for (key, value) in ns_properties.clone() {
@@ -163,22 +184,15 @@ impl Database for LanceNamespaceDatabase {
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> { async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
let mut table_id = request.namespace_path.clone(); let mut table_id = request.namespace_path.clone();
table_id.push(request.name.clone()); table_id.push(request.name.clone());
let describe_request = DescribeTableRequest {
id: Some(table_id.clone()),
..Default::default()
};
let describe_result = self.namespace.describe_table(describe_request).await;
match request.mode { match request.mode {
CreateTableMode::Create => { CreateTableMode::Create => {}
if describe_result.is_ok() {
return Err(Error::TableAlreadyExists {
name: request.name.clone(),
});
}
}
CreateTableMode::Overwrite => { CreateTableMode::Overwrite => {
let describe_request = DescribeTableRequest {
id: Some(table_id.clone()),
..Default::default()
};
let describe_result = self.namespace.describe_table(describe_request).await;
if describe_result.is_ok() { if describe_result.is_ok() {
// Drop the existing table - must succeed // Drop the existing table - must succeed
let drop_request = DropTableRequest { let drop_request = DropTableRequest {
@@ -194,6 +208,11 @@ impl Database for LanceNamespaceDatabase {
} }
} }
CreateTableMode::ExistOk(_) => { CreateTableMode::ExistOk(_) => {
let describe_request = DescribeTableRequest {
id: Some(table_id.clone()),
..Default::default()
};
let describe_result = self.namespace.describe_table(describe_request).await;
if describe_result.is_ok() { if describe_result.is_ok() {
let native_table = NativeTable::open_from_namespace( let native_table = NativeTable::open_from_namespace(
self.namespace.clone(), self.namespace.clone(),
@@ -221,7 +240,26 @@ impl Database for LanceNamespaceDatabase {
}; };
let (location, initial_storage_options, managed_versioning) = { let (location, initial_storage_options, managed_versioning) = {
let response = self.namespace.declare_table(declare_request).await?; let response = self
.namespace
.declare_table(declare_request)
.await
.map_err(|e| {
let err_str = e.to_string();
if matches!(request.mode, CreateTableMode::Create)
&& (err_str.contains("already exists")
|| err_str.contains("TableAlreadyExists")
|| err_str.contains("table already exists"))
{
Error::TableAlreadyExists {
name: request.name.clone(),
}
} else {
Error::Runtime {
message: format!("Failed to declare table: {}", e),
}
}
})?;
let loc = response.location.ok_or_else(|| Error::Runtime { let loc = response.location.ok_or_else(|| Error::Runtime {
message: "Table location is missing from declare_table response".to_string(), message: "Table location is missing from declare_table response".to_string(),
})?; })?;
+9 -9
View File
@@ -47,7 +47,7 @@ use std::format;
use std::path::Path; use std::path::Path;
use std::sync::Arc; use std::sync::Arc;
use crate::connection::PushdownOperation; use crate::connection::NamespaceClientPushdownOperation;
use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions}; use crate::data::scannable::{PeekedScannable, Scannable, estimate_write_partitions};
use crate::database::Database; use crate::database::Database;
@@ -1272,7 +1272,7 @@ pub struct NativeTable {
pub(crate) namespace_client: Option<Arc<dyn LanceNamespace>>, pub(crate) namespace_client: Option<Arc<dyn LanceNamespace>>,
// Operations to push down to the namespace server. // Operations to push down to the namespace server.
// pub(crate) so query.rs can access the field for server-side query execution. // pub(crate) so query.rs can access the field for server-side query execution.
pub(crate) pushdown_operations: HashSet<PushdownOperation>, pub(crate) pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
} }
impl std::fmt::Debug for NativeTable { impl std::fmt::Debug for NativeTable {
@@ -1359,7 +1359,7 @@ impl NativeTable {
params: Option<ReadParams>, params: Option<ReadParams>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
namespace_client: Option<Arc<dyn LanceNamespace>>, namespace_client: Option<Arc<dyn LanceNamespace>>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
managed_versioning: Option<bool>, managed_versioning: Option<bool>,
) -> Result<Self> { ) -> Result<Self> {
let params = params.unwrap_or_default(); let params = params.unwrap_or_default();
@@ -1470,7 +1470,7 @@ impl NativeTable {
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>, write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
params: Option<ReadParams>, params: Option<ReadParams>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
session: Option<Arc<lance::session::Session>>, session: Option<Arc<lance::session::Session>>,
) -> Result<Self> { ) -> Result<Self> {
let mut params = params.unwrap_or_default(); let mut params = params.unwrap_or_default();
@@ -1518,7 +1518,7 @@ impl NativeTable {
let id = Self::build_id(&namespace, name); let id = Self::build_id(&namespace, name);
let stored_namespace_client = let stored_namespace_client =
if pushdown_operations.contains(&PushdownOperation::QueryTable) { if pushdown_operations.contains(&NamespaceClientPushdownOperation::QueryTable) {
Some(namespace_client) Some(namespace_client)
} else { } else {
None None
@@ -1588,7 +1588,7 @@ impl NativeTable {
params: Option<WriteParams>, params: Option<WriteParams>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
namespace_client: Option<Arc<dyn LanceNamespace>>, namespace_client: Option<Arc<dyn LanceNamespace>>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
) -> Result<Self> { ) -> Result<Self> {
// Default params uses format v1. // Default params uses format v1.
let params = params.unwrap_or(WriteParams { let params = params.unwrap_or(WriteParams {
@@ -1635,7 +1635,7 @@ impl NativeTable {
params: Option<WriteParams>, params: Option<WriteParams>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
namespace_client: Option<Arc<dyn LanceNamespace>>, namespace_client: Option<Arc<dyn LanceNamespace>>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
) -> Result<Self> { ) -> Result<Self> {
let data: Box<dyn Scannable> = Box::new(RecordBatch::new_empty(schema)); let data: Box<dyn Scannable> = Box::new(RecordBatch::new_empty(schema));
Self::create( Self::create(
@@ -1685,7 +1685,7 @@ impl NativeTable {
write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>, write_store_wrapper: Option<Arc<dyn WrappingObjectStore>>,
params: Option<WriteParams>, params: Option<WriteParams>,
read_consistency_interval: Option<std::time::Duration>, read_consistency_interval: Option<std::time::Duration>,
pushdown_operations: HashSet<PushdownOperation>, pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
session: Option<Arc<lance::session::Session>>, session: Option<Arc<lance::session::Session>>,
) -> Result<Self> { ) -> Result<Self> {
// Build table_id from namespace + name for the storage options provider // Build table_id from namespace + name for the storage options provider
@@ -1738,7 +1738,7 @@ impl NativeTable {
let id = Self::build_id(&namespace, name); let id = Self::build_id(&namespace, name);
let stored_namespace_client = let stored_namespace_client =
if pushdown_operations.contains(&PushdownOperation::QueryTable) { if pushdown_operations.contains(&NamespaceClientPushdownOperation::QueryTable) {
Some(namespace_client) Some(namespace_client)
} else { } else {
None None
+8 -1
View File
@@ -285,7 +285,10 @@ mod tests {
use super::*; use super::*;
use crate::{connect, io::object_store::io_tracking::IoStatsHolder, table::WriteOptions}; use crate::{
connect, io::object_store::io_tracking::IoStatsHolder, table::WriteOptions,
utils::background_cache::clock,
};
async fn create_test_dataset(uri: &str) -> Dataset { async fn create_test_dataset(uri: &str) -> Dataset {
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
@@ -463,6 +466,10 @@ mod tests {
let uri = dir.path().to_str().unwrap(); let uri = dir.path().to_str().unwrap();
let ds = create_test_dataset(uri).await; let ds = create_test_dataset(uri).await;
// Other tests use a thread-local mock clock. Simulate leaked state from a
// previous test to ensure this wrapper starts from real time.
clock::advance_by(Duration::from_secs(60));
let wrapper = DatasetConsistencyWrapper::new_latest(ds, Some(Duration::from_millis(200))); let wrapper = DatasetConsistencyWrapper::new_latest(ds, Some(Duration::from_millis(200)));
// Populate the cache // Populate the cache
+2 -2
View File
@@ -4,7 +4,7 @@
use std::sync::Arc; use std::sync::Arc;
use super::NativeTable; use super::NativeTable;
use crate::connection::PushdownOperation; use crate::connection::NamespaceClientPushdownOperation;
use crate::error::{Error, Result}; use crate::error::{Error, Result};
use crate::expr::expr_to_sql_string; use crate::expr::expr_to_sql_string;
use crate::query::{ use crate::query::{
@@ -44,7 +44,7 @@ pub async fn execute_query(
// If QueryTable pushdown is enabled and namespace client is configured, use server-side query execution // If QueryTable pushdown is enabled and namespace client is configured, use server-side query execution
if table if table
.pushdown_operations .pushdown_operations
.contains(&PushdownOperation::QueryTable) .contains(&NamespaceClientPushdownOperation::QueryTable)
&& let Some(ref namespace_client) = table.namespace_client && let Some(ref namespace_client) = table.namespace_client
{ {
return execute_namespace_query(table, namespace_client.clone(), query, options).await; return execute_namespace_query(table, namespace_client.clone(), query, options).await;
@@ -107,6 +107,14 @@ where
refresh_window < ttl, refresh_window < ttl,
"refresh_window ({refresh_window:?}) must be less than ttl ({ttl:?})" "refresh_window ({refresh_window:?}) must be less than ttl ({ttl:?})"
); );
#[cfg(test)]
{
// Tests may advance the thread-local mock clock and leave it behind for
// the next test that happens to run on the same worker thread. Each new
// cache should start from a clean clock state instead of inheriting
// unrelated mock time from a previous test.
clock::clear_mock();
}
Self { Self {
inner: Arc::new(Mutex::new(CacheInner { inner: Arc::new(Mutex::new(CacheInner {
state: State::Empty, state: State::Empty,