Compare commits

..

13 Commits

Author SHA1 Message Date
Gatefixer 9ba03810e7 fix: forward commit outcome capabilities 2026-08-13 16:52:23 +00:00
Gatefixer b4053059bf Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2623-1 2026-08-13 16:03:12 +00:00
LanceDB Robot 4b7325bd74 chore: update lance dependency to v11.0.0-beta.8 (#3928)
Updates the Rust workspace and Java lance-core dependency to Lance
v11.0.0-beta.8, with refreshed Cargo lockfile metadata. No compatibility
fixes were required. Lance tag:
https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.8
2026-08-14 00:00:18 +08:00
Gatefixer 7a7b7a3941 Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2623-1
# Conflicts:
#	rust/lancedb/src/database/listing.rs
2026-08-13 15:53:12 +00:00
Gatefixer d52940cdab fix: guard UNC namespace client exports 2026-08-06 13:33:39 +00:00
Gatefixer fafc297675 fix: isolate namespace and clone target handlers 2026-08-06 12:33:34 +00:00
Gatefixer 82ebddbc10 test(rust): preserve UNC authority in lifecycle test 2026-08-06 12:07:38 +00:00
Gatefixer f114bba752 fix: preserve routed Windows store operations 2026-08-06 11:40:05 +00:00
Gatefixer 78024a30ce fix: preserve Windows local store contracts 2026-08-06 10:51:54 +00:00
Gatefixer 72500192e6 fix: share Windows stores across database tables 2026-08-06 10:26:12 +00:00
Gatefixer 8165857a50 Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2623-1
# Conflicts:
#	python/python/tests/test_db.py
2026-08-06 09:58:24 +00:00
Gatefixer 5fa98b9af8 fix: address Windows file store review feedback 2026-08-06 09:13:59 +00:00
Gatefixer b525cbbe6a fix: preserve UNC roots for local object stores 2026-08-05 20:44:50 +00:00
14 changed files with 1811 additions and 653 deletions
Generated
+43 -56
View File
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arc-swap",
"arrow",
@@ -4832,7 +4832,6 @@ dependencies = [
"async-recursion",
"async-trait",
"async_cell",
"aws-credential-types",
"aws-sdk-dynamodb",
"byteorder",
"bytes",
@@ -4848,7 +4847,6 @@ dependencies = [
"either",
"fst",
"futures",
"half",
"humantime",
"itertools 0.14.0",
"lance-arrow",
@@ -4890,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4913,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4927,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4936,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrayref",
"crunchy",
@@ -4947,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4956,12 +4954,10 @@ dependencies = [
"arrow-schema",
"async-trait",
"blake3",
"byteorder",
"bytes",
"datafusion-common",
"datafusion-sql",
"futures",
"itertools 0.14.0",
"lance-arrow",
"lance-derive",
"libc",
@@ -4979,7 +4975,6 @@ dependencies = [
"snafu 0.9.0",
"tempfile",
"tokio",
"tokio-stream",
"tokio-util",
"tracing",
"twox-hash",
@@ -4988,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow",
"arrow-array",
@@ -5019,8 +5014,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow",
"arrow-array",
@@ -5037,8 +5032,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"proc-macro2",
"quote",
@@ -5047,8 +5042,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5073,7 +5068,6 @@ dependencies = [
"num-traits",
"prost",
"prost-build",
"rand 0.9.5",
"tokio",
"tracing",
"xxhash-rust",
@@ -5082,8 +5076,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5114,8 +5108,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arc-swap",
"arrow",
@@ -5130,7 +5124,6 @@ dependencies = [
"async-trait",
"bitvec",
"bytes",
"chrono",
"crossbeam-queue",
"datafusion",
"datafusion-common",
@@ -5148,7 +5141,6 @@ dependencies = [
"lance-bitpacking",
"lance-core",
"lance-datafusion",
"lance-datagen",
"lance-encoding",
"lance-file",
"lance-index-core",
@@ -5177,13 +5169,12 @@ dependencies = [
"tempfile",
"tokio",
"tracing",
"uuid",
]
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5205,8 +5196,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow",
"arrow-array",
@@ -5220,7 +5211,6 @@ dependencies = [
"futures",
"http 1.5.0",
"io-uring",
"lance-arrow",
"lance-core",
"lance-namespace",
"log",
@@ -5238,29 +5228,28 @@ dependencies = [
"tokio",
"tracing",
"url",
"uuid",
]
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-schema",
"cc",
"half",
"lance-arrow",
"lance-core",
"num-traits",
"rand 0.9.5",
"rayon",
]
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow",
"async-trait",
@@ -5272,8 +5261,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5326,14 +5315,13 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-schema",
"byteorder",
"bytes",
"itertools 0.14.0",
"lance-core",
"roaring",
@@ -5342,8 +5330,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow",
"arrow-array",
@@ -5383,8 +5371,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5397,8 +5385,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.7#e581c49338bc83baf1ea50c5e235bd702f3fbeea"
version = "11.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.8#9acbac748e8a7d616146dac8b06da42d8e7c6b62"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5495,7 +5483,6 @@ dependencies = [
"urlencoding",
"uuid",
"walkdir",
"windows-sys 0.61.2",
]
[[package]]
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.7", default-features = false, "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.7", default-features = false, "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.7", default-features = false, "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.7", "tag" = "v11.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.8", default-features = false, "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.8", default-features = false, "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.8", default-features = false, "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.8", "tag" = "v11.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>11.0.0-beta.7</lance-core.version>
<lance-core.version>11.0.0-beta.8</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+2 -13
View File
@@ -716,20 +716,9 @@ class LanceDBConnection(DBConnection):
if not isinstance(uri, Path):
scheme = get_uri_scheme(uri)
is_local = isinstance(uri, Path) or scheme == "file"
if is_local:
is_file_uri = isinstance(uri, str) and uri.lower().startswith("file:")
if is_local and not is_file_uri:
if isinstance(uri, str):
# Strip file:// or file:/ scheme if present
# file:///path becomes file:/path after URL normalization
if uri.startswith("file://"):
uri = uri[7:] # Remove "file://"
elif uri.startswith("file:/"):
uri = uri[5:] # Remove "file:"
if sys.platform == "win32":
# On Windows, a path like /C:/path should become C:/path
if len(uri) >= 3 and uri[0] == "/" and uri[2] == ":":
uri = uri[1:]
uri = Path(uri)
uri = uri.expanduser().absolute()
Path(uri).mkdir(parents=True, exist_ok=True)
+2 -22
View File
@@ -919,17 +919,7 @@ class LanceNamespaceDBConnection(DBConnection):
The namespace client for this connection.
"""
if self._namespace_client is None:
if (
self._namespace_client_impl is None
or self._namespace_client_properties is None
):
raise ValueError(
"Cannot construct a Python namespace client without "
"namespace implementation properties"
)
self._namespace_client = namespace_connect(
self._namespace_client_impl, self._namespace_client_properties
)
self._namespace_client = LOOP.run(self._inner.namespace_client())
return self._namespace_client
@@ -1370,17 +1360,7 @@ class AsyncLanceNamespaceDBConnection:
The namespace client for this connection.
"""
if self._namespace_client is None:
if (
self._namespace_client_impl is None
or self._namespace_client_properties is None
):
raise ValueError(
"Cannot construct a Python namespace client without "
"namespace implementation properties"
)
self._namespace_client = namespace_connect(
self._namespace_client_impl, self._namespace_client_properties
)
self._namespace_client = await self._inner.namespace_client()
return self._namespace_client
+89
View File
@@ -89,6 +89,32 @@ def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeyp
assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
def test_connect_preserves_file_uri_authority(monkeypatch):
uri = "file://server/share/database"
received = []
async def fake_connect(passed_uri, *_args):
received.append(passed_uri)
return SimpleNamespace(uri=passed_uri)
monkeypatch.setattr("lancedb.db.lancedb_connect", fake_connect)
db = lancedb.connect(uri)
assert received == [uri]
assert db.uri == uri
def test_connect_file_uri_lifecycle(tmp_path):
uri = (tmp_path / "sync").as_uri()
db = lancedb.connect(uri)
db.create_table("test", data=[{"id": 1}])
assert db.table_names() == ["test"]
assert db.open_table("test").count_rows() == 1
db.drop_table("test")
assert db.table_names() == []
def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkeypatch):
from lancedb.background_loop import LOOP
from lancedb.db import LanceDBConnection
@@ -405,6 +431,35 @@ async def test_connect(tmp_path):
assert str(db) == f"ListingDatabase(uri={tmp_path}, read_consistency_interval=5s)"
@pytest.mark.asyncio
async def test_connect_async_preserves_file_uri_authority(monkeypatch):
uri = "file://server/share/database"
received = []
async def fake_connect(passed_uri, *_args):
received.append(passed_uri)
return SimpleNamespace(uri=passed_uri)
monkeypatch.setattr(lancedb, "lancedb_connect", fake_connect)
db = await lancedb.connect_async(uri)
assert received == [uri]
assert db.uri == uri
@pytest.mark.asyncio
async def test_connect_async_file_uri_lifecycle(tmp_path):
uri = (tmp_path / "async").as_uri()
db = await lancedb.connect_async(uri)
await db.create_table("test", data=[{"id": 1}])
assert await db.table_names() == ["test"]
table = await db.open_table("test")
assert await table.count_rows() == 1
await db.drop_table("test")
assert await db.table_names() == []
@pytest.mark.asyncio
async def test_close(mem_db_async: lancedb.AsyncConnection):
assert mem_db_async.is_open()
@@ -1180,6 +1235,40 @@ def test_clone_table_deep_clone_fails(tmp_path):
db.clone_table("cloned", source_uri, is_shallow=False)
class _UnsupportedNamespaceConfig:
async def namespace_client_config(self):
raise RuntimeError("UNC namespace client export is not supported")
def test_sync_namespace_client_propagates_export_guard(monkeypatch):
from lancedb.db import AsyncConnection, LanceDBConnection
monkeypatch.setattr(
"lancedb.db.namespace_connect",
lambda *_args, **_kwargs: pytest.fail("guarded config was reconstructed"),
)
db = LanceDBConnection.__new__(LanceDBConnection)
db._conn = AsyncConnection(_UnsupportedNamespaceConfig())
db._cached_namespace_client = None
with pytest.raises(RuntimeError, match="UNC namespace client export"):
db.namespace_client()
@pytest.mark.asyncio
async def test_async_namespace_client_propagates_export_guard(monkeypatch):
from lancedb.db import AsyncConnection
monkeypatch.setattr(
"lancedb.db.namespace_connect",
lambda *_args, **_kwargs: pytest.fail("guarded config was reconstructed"),
)
db = AsyncConnection(_UnsupportedNamespaceConfig())
with pytest.raises(RuntimeError, match="UNC namespace client export"):
await db.namespace_client()
@pytest.mark.skipif(sys.platform == "win32", reason="Namespace client issues")
def test_namespace_client_native_storage(tmp_path):
"""Test namespace_client() returns DirectoryNamespace for native storage."""
+41
View File
@@ -60,6 +60,11 @@ class _NamespaceClient:
return _ipc_file()
class _UnsupportedNamespaceConfig:
async def namespace_client_config(self):
raise RuntimeError("UNC namespace client export is not supported")
def _namespace_lance_table(namespace_client: _NamespaceClient) -> LanceTable:
table = LanceTable.__new__(LanceTable)
table._table = _FailingSyncInner()
@@ -138,6 +143,24 @@ class TestNamespaceConnection:
db.drop_namespace(["test_ns"])
assert "test_ns" not in db.list_namespaces().namespaces
def test_sync_namespace_client_propagates_export_guard(self, monkeypatch):
from lancedb.db import AsyncConnection
monkeypatch.setattr(
"lancedb.namespace.namespace_connect",
lambda *_args, **_kwargs: pytest.fail("guarded config was reconstructed"),
)
db = lancedb.LanceNamespaceDBConnection.__new__(
lancedb.LanceNamespaceDBConnection
)
db._namespace_client = None
db._namespace_client_impl = "dir"
db._namespace_client_properties = {"root": "file://server/share/database"}
db._inner = AsyncConnection(_UnsupportedNamespaceConfig())
with pytest.raises(RuntimeError, match="UNC namespace client export"):
db.namespace_client()
def test_create_table_through_namespace(self):
"""Test creating a table through namespace."""
db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
@@ -639,6 +662,24 @@ class TestAsyncNamespaceConnection:
await db.drop_namespace(["test_ns"])
assert "test_ns" not in (await db.list_namespaces()).namespaces
async def test_async_namespace_client_propagates_export_guard(self, monkeypatch):
from lancedb.db import AsyncConnection
monkeypatch.setattr(
"lancedb.namespace.namespace_connect",
lambda *_args, **_kwargs: pytest.fail("guarded config was reconstructed"),
)
db = lancedb.AsyncLanceNamespaceDBConnection.__new__(
lancedb.AsyncLanceNamespaceDBConnection
)
db._namespace_client = None
db._namespace_client_impl = "dir"
db._namespace_client_properties = {"root": "file://server/share/database"}
db._inner = AsyncConnection(_UnsupportedNamespaceConfig())
with pytest.raises(RuntimeError, match="UNC namespace client export"):
await db.namespace_client()
async def test_async_namespace_client_is_lazy(self):
"""namespace_client() should still return the backing client on demand."""
pytest.importorskip("lance")
-5
View File
@@ -115,11 +115,6 @@ serial_test = "3"
[target.'cfg(unix)'.dev-dependencies]
pprof = { version = "0.14", features = ["flamegraph"] }
[target.'cfg(windows)'.dependencies]
windows-sys = { version = "0.61", features = [
"Win32_Foundation",
"Win32_Storage_FileSystem",
] }
[features]
default = []
File diff suppressed because it is too large Load Diff
+132
View File
@@ -59,6 +59,46 @@ fn is_table_already_exists_namespace_error(err: &lance::Error) -> bool {
/// via the `delimiter` property.
const DEFAULT_NAMESPACE_DELIMITER: &str = "$";
#[cfg(any(windows, test))]
fn is_unc_root(root: &str) -> bool {
let normalized = root.replace('\\', "/");
let lowercase = normalized.to_ascii_lowercase();
if lowercase.starts_with("//?/unc/") {
return true;
}
if lowercase.starts_with("//?/") || lowercase.starts_with("//./") {
return false;
}
if let Some(authority) = normalized.strip_prefix("//") {
return !authority.is_empty() && !authority.starts_with('/');
}
normalized.split_once("://").is_some_and(|(scheme, path)| {
scheme.eq_ignore_ascii_case("file") && !path.is_empty() && !path.starts_with('/')
})
}
#[cfg(any(windows, test))]
fn is_directory_unc_root(ns_impl: &str, ns_properties: &HashMap<String, String>) -> bool {
ns_impl.eq_ignore_ascii_case("dir")
&& ns_properties
.get("root")
.is_some_and(|root| is_unc_root(root))
}
#[cfg(any(windows, test))]
#[derive(Debug)]
struct UnavailableUncDirectoryNamespace;
#[cfg(any(windows, test))]
#[async_trait]
impl LanceNamespace for UnavailableUncDirectoryNamespace {
fn namespace_id(&self) -> String {
"unavailable-unc-directory".to_string()
}
}
/// A database implementation that uses lance-namespace for table management
pub struct LanceNamespaceDatabase {
namespace: Arc<dyn LanceNamespace>,
@@ -92,6 +132,17 @@ fn resolve_delimiter(ns_properties: &HashMap<String, String>) -> String {
}
impl LanceNamespaceDatabase {
fn ensure_storage_supported(&self) -> Result<()> {
#[cfg(any(windows, test))]
if is_directory_unc_root(&self.ns_impl, &self.ns_properties) {
return Err(Error::NotSupported {
message: "Directory namespace operations are not supported for UNC roots because the namespace manifest resolver does not preserve file URI authorities; use flat root tables or a non-UNC namespace root"
.to_string(),
});
}
Ok(())
}
pub fn from_namespace_client(
namespace_client: Arc<dyn LanceNamespace>,
namespace_client_impl: String,
@@ -153,6 +204,25 @@ impl LanceNamespaceDatabase {
pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
new_table_config: NewTableConfig,
) -> Result<Self> {
#[cfg(any(windows, test))]
if is_directory_unc_root(ns_impl, &ns_properties) {
let freshness_baselines: FreshnessBaselines = Arc::new(Mutex::new(HashMap::new()));
let delimiter = resolve_delimiter(&ns_properties);
return Ok(Self {
namespace: Arc::new(UnavailableUncDirectoryNamespace),
storage_options,
read_consistency_interval,
session,
uri: format!("namespace://{}", ns_impl),
pushdown_operations,
ns_impl: ns_impl.to_string(),
ns_properties,
new_table_config,
freshness_baselines,
delimiter,
});
}
let mut builder = ConnectBuilder::new(ns_impl);
for (key, value) in ns_properties.clone() {
builder = builder.property(key, value);
@@ -310,6 +380,7 @@ impl Database for LanceNamespaceDatabase {
&self,
request: ListNamespacesRequest,
) -> Result<ListNamespacesResponse> {
self.ensure_storage_supported()?;
Ok(self.namespace.list_namespaces(request).await?)
}
@@ -317,10 +388,12 @@ impl Database for LanceNamespaceDatabase {
&self,
request: CreateNamespaceRequest,
) -> Result<CreateNamespaceResponse> {
self.ensure_storage_supported()?;
Ok(self.namespace.create_namespace(request).await?)
}
async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result<DropNamespaceResponse> {
self.ensure_storage_supported()?;
Ok(self.namespace.drop_namespace(request).await?)
}
@@ -328,10 +401,12 @@ impl Database for LanceNamespaceDatabase {
&self,
request: DescribeNamespaceRequest,
) -> Result<DescribeNamespaceResponse> {
self.ensure_storage_supported()?;
Ok(self.namespace.describe_namespace(request).await?)
}
async fn table_names(&self, request: TableNamesRequest) -> Result<Vec<String>> {
self.ensure_storage_supported()?;
let ns_request = ListTablesRequest {
id: Some(request.namespace_path),
page_token: request.start_after,
@@ -345,10 +420,12 @@ impl Database for LanceNamespaceDatabase {
}
async fn list_tables(&self, request: ListTablesRequest) -> Result<ListTablesResponse> {
self.ensure_storage_supported()?;
Ok(self.namespace.list_tables(request).await?)
}
async fn create_table(&self, request: DbCreateTableRequest) -> Result<Arc<dyn BaseTable>> {
self.ensure_storage_supported()?;
let mut table_id = request.namespace_path.clone();
table_id.push(request.name.clone());
let mut existing_table = None;
@@ -518,6 +595,7 @@ impl Database for LanceNamespaceDatabase {
}
async fn open_table(&self, request: OpenTableRequest) -> Result<Arc<dyn BaseTable>> {
self.ensure_storage_supported()?;
let native_table = NativeTable::open_from_namespace(
self.namespace.clone(),
&request.name,
@@ -547,6 +625,7 @@ impl Database for LanceNamespaceDatabase {
cur_namespace_path: &[String],
new_namespace_path: &[String],
) -> Result<()> {
self.ensure_storage_supported()?;
let mut cur_table_id = cur_namespace_path.to_vec();
cur_table_id.push(cur_name.to_string());
@@ -573,6 +652,7 @@ impl Database for LanceNamespaceDatabase {
}
async fn drop_table(&self, name: &str, namespace_path: &[String]) -> Result<()> {
self.ensure_storage_supported()?;
let mut table_id = namespace_path.to_vec();
table_id.push(name.to_string());
@@ -612,10 +692,12 @@ impl Database for LanceNamespaceDatabase {
}
async fn namespace_client(&self) -> Result<Arc<dyn LanceNamespace>> {
self.ensure_storage_supported()?;
Ok(self.namespace.clone())
}
async fn namespace_client_config(&self) -> Result<(String, HashMap<String, String>)> {
self.ensure_storage_supported()?;
Ok((self.ns_impl.clone(), self.ns_properties.clone()))
}
}
@@ -644,6 +726,56 @@ mod tests {
RecordBatch::try_new(schema, vec![Arc::new(id_array), Arc::new(name_array)]).unwrap()
}
#[test]
fn detects_unc_namespace_roots_without_misclassifying_drives() {
for root in [
"file://server/share/database",
"FILE://server/share/database",
r"\\server\share\database",
r"\\?\UNC\server\share\database",
] {
assert!(is_unc_root(root), "expected an UNC root: {root}");
}
for root in [
"file:///C:/database",
r"C:\database",
r"\\?\C:\database",
"/var/lib/database",
] {
assert!(!is_unc_root(root), "expected a non-UNC root: {root}");
}
}
#[tokio::test]
async fn directory_namespace_fails_closed_before_accessing_an_unc_root() {
let mut properties = HashMap::new();
properties.insert(
"root".to_string(),
"file://server/share/database".to_string(),
);
let db = LanceNamespaceDatabase::connect(
"dir",
properties,
HashMap::new(),
None,
None,
HashSet::new(),
)
.await
.unwrap();
let error = db
.list_tables(ListTablesRequest::default())
.await
.unwrap_err();
assert!(matches!(error, Error::NotSupported { .. }));
assert!(error.to_string().contains("UNC roots"));
let error = db.namespace_client_config().await.unwrap_err();
assert!(matches!(error, Error::NotSupported { .. }));
assert!(error.to_string().contains("UNC roots"));
}
#[tokio::test]
async fn test_namespace_connection_simple() {
// Test that namespace connections work with simple connect_namespace(impl_type, properties)
+798 -3
View File
@@ -1,12 +1,22 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! A mirroring object store that mirror writes to a secondary object store
//! Object-store providers and adapters used by LanceDB.
use std::{fmt::Formatter, sync::Arc};
#[cfg(any(windows, test))]
use futures::TryStreamExt;
use futures::{StreamExt, TryFutureExt, stream::BoxStream};
use lance::io::WrappingObjectStore;
#[cfg(any(windows, test))]
use lance_table::{
format::{IndexMetadata, Manifest, Transaction},
io::commit::{
CommitError, CommitHandler, ManifestLocation, ManifestNamingScheme, ManifestWriter,
RenameCommitHandler,
},
};
use object_store::{
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
@@ -15,11 +25,796 @@ use object_store::{
use async_trait::async_trait;
#[cfg(any(windows, test))]
use lance_core::{Error as LanceError, Result as LanceResult};
#[cfg(test)]
use lance_io::object_store::ObjectStoreRegistry;
#[cfg(any(windows, test))]
use lance_io::object_store::{
DEFAULT_LOCAL_IO_PARALLELISM, ObjectStoreParams, ObjectStoreProvider, StorageOptions,
};
#[cfg(any(windows, test))]
use object_store::local::LocalFileSystem;
#[cfg(any(windows, test))]
use url::Url;
#[cfg(test)]
pub mod io_tracking;
#[cfg(windows)]
pub(crate) mod windows;
/// A local commit handler that resolves the latest manifest through the object store.
///
/// Lance's native local shortcut reconstructs the selected manifest from a
/// filesystem path. On Windows that conversion drops the authority from a UNC
/// path. Listing through the already-rooted object store preserves the structural
/// server/share prefix while retaining the normal atomic-rename commit behavior.
#[derive(Debug)]
#[cfg(any(windows, test))]
struct RootedFileCommitHandler;
#[cfg(any(windows, test))]
#[async_trait]
impl CommitHandler for RootedFileCommitHandler {
fn is_version_not_found_definitive(&self) -> bool {
RenameCommitHandler.is_version_not_found_definitive()
}
fn propagate_commit_error_after_success(&self) -> bool {
RenameCommitHandler.propagate_commit_error_after_success()
}
async fn resolve_latest_location(
&self,
base_path: &Path,
object_store: &lance::io::ObjectStore,
) -> LanceResult<ManifestLocation> {
self.list_manifest_locations(base_path, object_store, true)
.try_next()
.await?
.ok_or_else(|| LanceError::not_found(base_path.to_string()))
}
async fn commit(
&self,
manifest: &mut Manifest,
indices: Option<Vec<IndexMetadata>>,
base_path: &Path,
object_store: &lance::io::ObjectStore,
manifest_writer: ManifestWriter,
naming_scheme: ManifestNamingScheme,
transaction: Option<Transaction>,
) -> std::result::Result<ManifestLocation, CommitError> {
RenameCommitHandler
.commit(
manifest,
indices,
base_path,
object_store,
manifest_writer,
naming_scheme,
transaction,
)
.await
}
}
#[cfg(any(windows, test))]
pub(crate) fn rooted_file_commit_handler() -> Arc<dyn CommitHandler> {
Arc::new(RootedFileCommitHandler)
}
/// A file-store provider that anchors each request at its filesystem root.
///
/// On Windows, an unprefixed [`LocalFileSystem`] cannot service UNC paths. Its
/// conversion to an object-store [`Path`] drops the UNC host, so subsequent I/O
/// is directed at a different local path. Anchoring the store at the drive or
/// UNC-share root keeps the UNC authority in the filesystem prefix and exposes
/// only paths relative to that prefix to `object_store`.
///
/// Extracted paths retain the native drive or UNC-share root as a structural
/// first component. This keeps Lance's local classification and optimized I/O
/// safe without recovering absolute path provenance from ambiguous path text.
#[cfg(any(windows, test))]
#[derive(Debug)]
struct PrefixedFileStoreProvider;
#[cfg(any(windows, test))]
impl PrefixedFileStoreProvider {
fn root_and_relative_path(url: &Url) -> LanceResult<(std::path::PathBuf, Path)> {
let filesystem_path = url.to_file_path().map_err(|_| {
LanceError::invalid_input(format!("Unable to convert URL '{url}' to a local path"))
})?;
let mut root = std::path::PathBuf::new();
for component in filesystem_path.components() {
match component {
std::path::Component::Prefix(_) | std::path::Component::RootDir => {
root.push(component.as_os_str());
}
_ => break,
}
}
if root.as_os_str().is_empty() {
return Err(LanceError::invalid_input(format!(
"Local path '{}' has no filesystem root",
filesystem_path.display()
)));
}
let relative = filesystem_path.strip_prefix(&root).map_err(|_| {
LanceError::invalid_input(format!(
"Local path '{}' is not beneath store root '{}'",
filesystem_path.display(),
root.display()
))
})?;
let relative = relative
.components()
.filter_map(|component| match component {
std::path::Component::Normal(part) => Some(part),
_ => None,
})
.map(|part| {
part.to_str().ok_or_else(|| {
LanceError::invalid_input(format!(
"Local path '{}' is not valid UTF-8",
filesystem_path.display()
))
})
})
.collect::<LanceResult<Vec<_>>>()?
.join("/");
Ok((root, Path::parse(relative)?))
}
/// Preserve the native filesystem root as a structural path component.
///
/// `object_store::Path::from_absolute_path` converts a UNC path to its URL
/// path and loses the server. Keeping `C:` or `\\server\share` as the first
/// component distinguishes an absolute path from every relative path while
/// remaining directly usable by Windows filesystem APIs.
fn rooted_path(root: &std::path::Path, relative: &Path) -> LanceResult<Path> {
let root = root.to_string_lossy();
let root = root.trim_end_matches(['/', '\\']);
if root.is_empty() {
return Ok(relative.clone());
}
Ok(relative
.parts()
.fold(Path::parse(root)?, |path, part| path.join(part)))
}
}
/// A local store rooted at a Windows drive or UNC share.
///
/// Most calls use paths returned by [`PrefixedFileStoreProvider`], which are
/// relative to `root`. Some Lance operations retain an existing object store
/// while independently re-extracting a Windows file URI with the default file
/// provider. Those paths include the drive (`C:/...`) or UNC share
/// (`share/...`) again. Normalize that absolute alias before delegating so the
/// filesystem prefix is never applied twice.
#[cfg(any(windows, test))]
#[derive(Debug, Clone)]
struct RootedLocalFileSystem {
inner: Arc<LocalFileSystem>,
root: std::path::PathBuf,
absolute_alias: Path,
}
#[cfg(any(windows, test))]
impl RootedLocalFileSystem {
fn new(root: std::path::PathBuf) -> LanceResult<Self> {
let absolute_alias = PrefixedFileStoreProvider::rooted_path(&root, &Path::default())?;
Ok(Self {
inner: Arc::new(LocalFileSystem::new_with_prefix(&root)?),
root,
absolute_alias,
})
}
fn path_from_parts<'a>(parts: impl Iterator<Item = object_store::path::PathPart<'a>>) -> Path {
parts.fold(Path::default(), |path, part| path.join(part))
}
fn normalize(&self, path: &Path) -> Path {
if self.absolute_alias.as_ref().is_empty() {
return path.clone();
}
let Some(suffix) = path.prefix_match(&self.absolute_alias) else {
return path.clone();
};
Self::path_from_parts(suffix)
}
fn restore_prefix(&self, path: Path, requested: &Path, normalized: &Path) -> Path {
if requested == normalized {
return path;
}
path.prefix_match(normalized)
.map(|suffix| suffix.fold(requested.clone(), |path, part| path.join(part)))
.unwrap_or(path)
}
}
#[cfg(any(windows, test))]
impl std::fmt::Display for RootedLocalFileSystem {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "RootedLocalFileSystem({})", self.root.display())
}
}
#[cfg(any(windows, test))]
#[async_trait]
impl ObjectStore for RootedLocalFileSystem {
async fn put_opts(
&self,
location: &Path,
payload: PutPayload,
options: PutOptions,
) -> Result<PutResult> {
self.inner
.put_opts(&self.normalize(location), payload, options)
.await
}
async fn put_multipart_opts(
&self,
location: &Path,
options: PutMultipartOptions,
) -> Result<Box<dyn MultipartUpload>> {
self.inner
.put_multipart_opts(&self.normalize(location), options)
.await
}
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
let normalized = self.normalize(location);
let mut result = self.inner.get_opts(&normalized, options).await?;
result.meta.location = location.clone();
Ok(result)
}
fn delete_stream(
&self,
locations: BoxStream<'static, Result<Path>>,
) -> BoxStream<'static, Result<Path>> {
let store = self.clone();
locations
.map(move |location| {
let store = store.clone();
async move {
let location = location?;
let normalized = store.normalize(&location);
store.inner.delete(&normalized).await?;
Ok(location)
}
})
.buffered(10)
.boxed()
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
let requested = prefix.cloned().unwrap_or_default();
let normalized = self.normalize(&requested);
let store = self.clone();
self.inner
.list(prefix.map(|_| &normalized))
.map(move |result| {
result.map(|mut meta| {
meta.location = store.restore_prefix(meta.location, &requested, &normalized);
meta
})
})
.boxed()
}
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
let requested = prefix.cloned().unwrap_or_default();
let normalized = self.normalize(&requested);
let mut result = self
.inner
.list_with_delimiter(prefix.map(|_| &normalized))
.await?;
for meta in &mut result.objects {
meta.location = self.restore_prefix(meta.location.clone(), &requested, &normalized);
}
for path in &mut result.common_prefixes {
*path = self.restore_prefix(path.clone(), &requested, &normalized);
}
Ok(result)
}
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
self.inner
.copy_opts(&self.normalize(from), &self.normalize(to), options)
.await
}
}
#[cfg(any(windows, test))]
#[async_trait]
impl ObjectStoreProvider for PrefixedFileStoreProvider {
async fn new_store(
&self,
base_path: Url,
params: &ObjectStoreParams,
) -> LanceResult<lance::io::ObjectStore> {
let (root, _) = Self::root_and_relative_path(&base_path)?;
let raw_store: Arc<dyn ObjectStore> = Arc::new(RootedLocalFileSystem::new(root)?);
// Native local shortcuts are safe only when the rooted filesystem is
// the final store. An arbitrary wrapper can redirect I/O, so use
// Lance's non-cloud object-store route in that case. This keeps local
// scan planning without enabling native copy/delete or the io_uring
// scheduler, all of which would bypass the wrapper.
let location = if params.object_store_wrapper.is_some() {
Url::parse("memory:///").expect("static URL must be valid")
} else {
Url::parse("file:///").expect("static URL must be valid")
};
let storage_options =
StorageOptions::new(params.storage_options().cloned().unwrap_or_default());
// ObjectStore::new initializes the private local-store fields. The
// registry owns tracing, custom wrapper, and I/O tracker installation,
// so restore the raw store before returning to avoid applying them twice.
let mut store = lance::io::ObjectStore::new(
raw_store.clone(),
location,
Some(params.block_size.unwrap_or(4 * 1024)),
None,
false,
false,
DEFAULT_LOCAL_IO_PARALLELISM,
storage_options.download_retry_count(),
params.storage_options(),
);
store.inner = raw_store;
store.store_prefix =
self.calculate_object_store_prefix(&base_path, params.storage_options())?;
Ok(store)
}
fn extract_path(&self, url: &Url) -> LanceResult<Path> {
let (root, relative) = Self::root_and_relative_path(url)?;
Self::rooted_path(&root, &relative)
}
fn calculate_object_store_prefix(
&self,
url: &Url,
_storage_options: Option<&std::collections::HashMap<String, String>>,
) -> LanceResult<String> {
let (root, _) = Self::root_and_relative_path(url)?;
let root = root.canonicalize()?;
// One store per drive or UNC share keeps registry and metrics
// cardinality bounded. Absolute-vs-relative provenance is carried by
// the extracted path instead of the cache key.
Ok(format!("file${}", root.display()))
}
}
/// Build LanceDB's default session with the Windows file fallback installed.
///
/// Callers that provide a Session retain its registry unchanged.
pub(crate) fn new_default_session() -> Arc<lance::session::Session> {
let session = Arc::new(lance::session::Session::default());
#[cfg(windows)]
session
.store_registry()
.insert("file", Arc::new(PrefixedFileStoreProvider));
session
}
#[cfg(test)]
pub(crate) fn new_prefixed_file_session() -> Arc<lance::session::Session> {
let session = Arc::new(lance::session::Session::default());
session
.store_registry()
.insert("file", Arc::new(PrefixedFileStoreProvider));
session
}
#[cfg(test)]
mod prefixed_file_store_test {
use super::*;
use object_store::memory::InMemory;
use std::sync::atomic::{AtomicUsize, Ordering};
#[derive(Debug, Default)]
struct CountingWrapper {
calls: AtomicUsize,
}
impl WrappingObjectStore for CountingWrapper {
fn wrap(
&self,
_store_prefix: &str,
original: Arc<dyn ObjectStore>,
) -> Arc<dyn ObjectStore> {
self.calls.fetch_add(1, Ordering::Relaxed);
original
}
}
#[derive(Debug, Default)]
struct MemoryRedirectWrapper {
store: Arc<InMemory>,
}
impl WrappingObjectStore for MemoryRedirectWrapper {
fn wrap(
&self,
_store_prefix: &str,
_original: Arc<dyn ObjectStore>,
) -> Arc<dyn ObjectStore> {
self.store.clone()
}
}
#[test]
fn commit_handler_forwards_commit_outcome_capabilities() {
let handler = RootedFileCommitHandler;
assert!(handler.is_version_not_found_definitive());
assert!(!handler.propagate_commit_error_after_success());
}
#[tokio::test]
async fn anchors_new_and_existing_directories_at_a_filesystem_prefix() {
let tempdir = tempfile::tempdir().unwrap();
let database_path = tempdir.path().join("database");
std::fs::create_dir(&database_path).unwrap();
let table_path = database_path.join("test.lance");
let table_url = Url::from_directory_path(&table_path).unwrap();
let registry = Arc::new(ObjectStoreRegistry::default());
registry.insert("file", Arc::new(PrefixedFileStoreProvider));
// The extracted path remains absolute for Lance's local fast paths,
// while the inner store strips the structural root before delegation.
let (store, base_path) = lance::io::ObjectStore::from_uri_and_params(
registry.clone(),
table_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
assert_eq!(store.scheme(), "file");
assert!(store.is_local());
assert!(!store.is_cloud());
assert_eq!(store.block_size(), 4 * 1024);
assert_eq!(store.io_parallelism(), DEFAULT_LOCAL_IO_PARALLELISM);
assert_eq!(base_path.filename(), Some("test.lance"));
let initial_base_path = base_path.clone();
let marker = base_path.join("marker");
store
.inner
.put(&marker, bytes::Bytes::from_static(b"new").into())
.await
.unwrap();
assert_eq!(std::fs::read(table_path.join("marker")).unwrap(), b"new");
// Once the table directory exists, a fresh store uses the same stable
// root and object-store path.
drop(store);
let (store, base_path) = lance::io::ObjectStore::from_uri_and_params(
registry,
table_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
assert_eq!(base_path, initial_base_path);
let contents = store
.inner
.get(&base_path.join("marker"))
.await
.unwrap()
.bytes()
.await
.unwrap();
assert_eq!(contents.as_ref(), b"new");
}
#[tokio::test]
async fn applies_wrapper_and_io_tracking_once() {
let tempdir = tempfile::tempdir().unwrap();
let table_url = Url::from_directory_path(tempdir.path().join("test.lance")).unwrap();
let registry = Arc::new(ObjectStoreRegistry::default());
registry.insert("file", Arc::new(PrefixedFileStoreProvider));
let wrapper = Arc::new(CountingWrapper::default());
let params = ObjectStoreParams {
object_store_wrapper: Some(wrapper.clone()),
..Default::default()
};
let (store, base_path) =
lance::io::ObjectStore::from_uri_and_params(registry, table_url.as_str(), &params)
.await
.unwrap();
assert_eq!(wrapper.calls.load(Ordering::Relaxed), 1);
assert_eq!(store.scheme(), "memory");
assert!(!store.is_local());
assert!(!store.is_cloud());
assert!(!store.prefers_lite_scheduler());
store
.inner
.put(
&base_path.join("marker"),
bytes::Bytes::from_static(b"tracked").into(),
)
.await
.unwrap();
let stats = store.io_tracker().stats();
assert_eq!(stats.write_iops, 1);
assert_eq!(stats.written_bytes, 7);
}
#[tokio::test]
async fn wrapped_store_cleanup_does_not_bypass_the_wrapper() {
let tempdir = tempfile::tempdir().unwrap();
let table_path = tempdir.path().join("test.lance");
std::fs::create_dir(&table_path).unwrap();
std::fs::write(table_path.join("native-marker"), b"native").unwrap();
let table_url = Url::from_directory_path(&table_path).unwrap();
let registry = Arc::new(ObjectStoreRegistry::default());
registry.insert("file", Arc::new(PrefixedFileStoreProvider));
let wrapper = Arc::new(MemoryRedirectWrapper::default());
let params = ObjectStoreParams {
object_store_wrapper: Some(wrapper.clone()),
..Default::default()
};
let (store, base_path) =
lance::io::ObjectStore::from_uri_and_params(registry, table_url.as_str(), &params)
.await
.unwrap();
let wrapped_marker = base_path.clone().join("wrapped-marker");
store
.inner
.put(
&wrapped_marker,
bytes::Bytes::from_static(b"wrapped").into(),
)
.await
.unwrap();
store.remove_dir_all(base_path).await.unwrap();
assert!(table_path.join("native-marker").exists());
assert!(matches!(
wrapper.store.head(&wrapped_marker).await,
Err(object_store::Error::NotFound { .. })
));
}
#[tokio::test]
async fn rooted_commit_handler_uses_store_paths_for_latest_manifest() {
let tempdir = tempfile::tempdir().unwrap();
let versions = tempdir.path().join("_versions");
std::fs::create_dir(&versions).unwrap();
std::fs::write(versions.join("2.manifest"), b"native").unwrap();
let base_path = Path::from_absolute_path(tempdir.path()).unwrap();
let raw_store: Arc<dyn ObjectStore> = Arc::new(InMemory::new());
raw_store
.put(
&base_path.clone().join("_versions").join("1.manifest"),
bytes::Bytes::from_static(b"wrapped").into(),
)
.await
.unwrap();
let mut store = lance::io::ObjectStore::new(
raw_store.clone(),
Url::parse("file:///").unwrap(),
Some(4 * 1024),
None,
false,
false,
DEFAULT_LOCAL_IO_PARALLELISM,
0,
None,
);
store.inner = raw_store;
let native = RenameCommitHandler
.resolve_latest_location(&base_path, &store)
.await
.unwrap();
assert_eq!(native.version, 2);
let rooted = rooted_file_commit_handler()
.resolve_latest_location(&base_path, &store)
.await
.unwrap();
assert_eq!(rooted.version, 1);
assert_eq!(
rooted.path,
base_path.clone().join("_versions").join("1.manifest")
);
}
#[tokio::test]
async fn reuses_store_cache_across_database_paths() {
let tempdir = tempfile::tempdir().unwrap();
let first_url = Url::from_directory_path(tempdir.path().join("database")).unwrap();
let second_url = Url::from_directory_path(tempdir.path().join("share/database")).unwrap();
let registry = Arc::new(ObjectStoreRegistry::default());
registry.insert("file", Arc::new(PrefixedFileStoreProvider));
let (first, _) = lance::io::ObjectStore::from_uri_and_params(
registry.clone(),
first_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
let (second, _) = lance::io::ObjectStore::from_uri_and_params(
registry.clone(),
second_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
assert!(Arc::ptr_eq(&first, &second));
assert_eq!(registry.stats().misses, 1);
assert_eq!(registry.stats().hits, 1);
}
#[tokio::test]
async fn reuses_database_store_for_table_and_clone_targets() {
let tempdir = tempfile::tempdir().unwrap();
let database_url = Url::from_directory_path(tempdir.path().join("database")).unwrap();
let source_url =
Url::from_directory_path(tempdir.path().join("database/source.lance")).unwrap();
let target_url =
Url::from_directory_path(tempdir.path().join("database/target.lance")).unwrap();
let registry = Arc::new(ObjectStoreRegistry::default());
registry.insert("file", Arc::new(PrefixedFileStoreProvider));
let (database, _) = lance::io::ObjectStore::from_uri_and_params(
registry.clone(),
database_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
let (source, _) = lance::io::ObjectStore::from_uri_and_params(
registry.clone(),
source_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
let (target, _) = lance::io::ObjectStore::from_uri_and_params(
registry.clone(),
target_url.as_str(),
&ObjectStoreParams::default(),
)
.await
.unwrap();
assert!(Arc::ptr_eq(&database, &source));
assert!(Arc::ptr_eq(&database, &target));
assert_eq!(registry.stats().misses, 1);
assert_eq!(registry.stats().hits, 2);
}
#[test]
fn normalizes_absolute_drive_and_unc_aliases() {
let tempdir = tempfile::tempdir().unwrap();
let mut drive_store = RootedLocalFileSystem::new(tempdir.path().to_path_buf()).unwrap();
drive_store.absolute_alias = Path::from("C:");
assert_eq!(
drive_store.normalize(&Path::from("C:/Users/db/table.lance")),
Path::from("Users/db/table.lance")
);
let mut unc_store = RootedLocalFileSystem::new(tempdir.path().to_path_buf()).unwrap();
unc_store.absolute_alias = Path::parse(r"\\server\share").unwrap();
assert_eq!(
unc_store.normalize(&Path::parse(r"\\server\share/share/db/table.lance").unwrap()),
Path::from("share/db/table.lance")
);
assert_eq!(
unc_store.normalize(&Path::from("share/db/table.lance")),
Path::from("share/db/table.lance")
);
}
#[test]
fn relative_unc_alias_does_not_cross_database_roots() {
let tempdir = tempfile::tempdir().unwrap();
let mut store = RootedLocalFileSystem::new(tempdir.path().to_path_buf()).unwrap();
store.absolute_alias = Path::parse(r"\\server\share").unwrap();
let relative = Path::from("share/database/table.lance/marker");
assert_eq!(store.normalize(&relative), relative);
assert_eq!(
store.normalize(
&Path::parse(r"\\server\share/share/database/table.lance/marker").unwrap()
),
relative
);
}
#[tokio::test]
async fn routes_unc_alias_lifecycle_through_the_prefix() {
let tempdir = tempfile::tempdir().unwrap();
let mut store = RootedLocalFileSystem::new(tempdir.path().to_path_buf()).unwrap();
store.absolute_alias = Path::parse(r"\\server\share").unwrap();
let table = Path::parse(r"\\server\share/share/db/test.lance").unwrap();
let marker = table.clone().join("marker");
store
.put(&marker, bytes::Bytes::from_static(b"unc").into())
.await
.unwrap();
assert_eq!(
std::fs::read(tempdir.path().join("share/db/test.lance/marker")).unwrap(),
b"unc"
);
let listed = store.list(Some(&table)).collect::<Vec<_>>().await;
assert_eq!(listed.len(), 1);
assert_eq!(listed[0].as_ref().unwrap().location, marker);
assert_eq!(
store
.get(&marker)
.await
.unwrap()
.bytes()
.await
.unwrap()
.as_ref(),
b"unc"
);
store.delete(&marker).await.unwrap();
assert!(!tempdir.path().join("share/db/test.lance/marker").exists());
}
#[cfg(windows)]
#[test]
fn extracts_drive_and_unc_share_roots() {
let (root, path) = PrefixedFileStoreProvider::root_and_relative_path(
&Url::parse("file:///C:/database").unwrap(),
)
.unwrap();
assert_eq!(root, std::path::PathBuf::from(r"C:\"));
assert_eq!(path, Path::from("database"));
let (root, path) = PrefixedFileStoreProvider::root_and_relative_path(
&Url::parse("file://server/share/database").unwrap(),
)
.unwrap();
assert_eq!(root, std::path::PathBuf::from(r"\\server\share\"));
assert_eq!(path, Path::from("database"));
}
#[test]
fn preserves_drive_and_unc_roots_in_extracted_paths() {
assert_eq!(
PrefixedFileStoreProvider::rooted_path(
std::path::Path::new(r"C:\"),
&Path::from("database/table.lance")
)
.unwrap(),
Path::from("C:/database/table.lance")
);
assert_eq!(
PrefixedFileStoreProvider::rooted_path(
std::path::Path::new(r"\\server\share\"),
&Path::from("database/table.lance")
)
.unwrap(),
Path::parse(r"\\server\share/database/table.lance").unwrap()
);
}
}
#[derive(Debug)]
struct MirroringObjectStore {
-223
View File
@@ -1,223 +0,0 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Windows local filesystem compatibility for atomic manifest commits.
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::os::windows::ffi::OsStrExt;
use std::path::{Path as StdPath, PathBuf};
use std::sync::Arc;
use bytes::Bytes;
use futures::stream::BoxStream;
use lance::io::WrappingObjectStore;
use object_store::{
CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions,
RenameTargetMode, Result, UploadPart, path::Path,
};
use windows_sys::Win32::Foundation::{ERROR_ALREADY_EXISTS, ERROR_FILE_EXISTS};
use windows_sys::Win32::Storage::FileSystem::MoveFileExW;
const STORE_NAME: &str = "WindowsLocalFileSystem";
/// Uses the Windows move primitive for create-only renames on local stores.
///
/// `object_store` implements create-only local renames with a hard link followed
/// by a delete. Some Windows filesystems do not support hard links, but
/// `MoveFileExW` without `MOVEFILE_REPLACE_EXISTING` provides the same atomic
/// create-only rename semantics without requiring them.
#[derive(Debug, Default)]
pub struct WindowsLocalFileSystemWrapper;
impl WrappingObjectStore for WindowsLocalFileSystemWrapper {
fn wrap(&self, _store_prefix: &str, target: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
Arc::new(WindowsLocalFileSystem { target })
}
}
#[derive(Debug)]
struct WindowsLocalFileSystem {
target: Arc<dyn ObjectStore>,
}
impl Display for WindowsLocalFileSystem {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{STORE_NAME}({})", self.target)
}
}
#[async_trait::async_trait]
#[deny(clippy::missing_trait_methods)]
impl ObjectStore for WindowsLocalFileSystem {
async fn put_opts(
&self,
location: &Path,
bytes: PutPayload,
opts: PutOptions,
) -> Result<PutResult> {
self.target.put_opts(location, bytes, opts).await
}
async fn put_multipart_opts(
&self,
location: &Path,
opts: PutMultipartOptions,
) -> Result<Box<dyn MultipartUpload>> {
self.target.put_multipart_opts(location, opts).await
}
async fn get_opts(&self, location: &Path, options: GetOptions) -> Result<GetResult> {
self.target.get_opts(location, options).await
}
async fn get_ranges(
&self,
location: &Path,
ranges: &[std::ops::Range<u64>],
) -> Result<Vec<Bytes>> {
self.target.get_ranges(location, ranges).await
}
fn delete_stream(
&self,
locations: BoxStream<'static, Result<Path>>,
) -> BoxStream<'static, Result<Path>> {
self.target.delete_stream(locations)
}
fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
self.target.list(prefix)
}
fn list_with_offset(
&self,
prefix: Option<&Path>,
offset: &Path,
) -> BoxStream<'static, Result<ObjectMeta>> {
self.target.list_with_offset(prefix, offset)
}
async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result<ListResult> {
self.target.list_with_delimiter(prefix).await
}
async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
self.target.copy_opts(from, to, options).await
}
async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> Result<()> {
if options.target_mode != RenameTargetMode::Create {
return self.target.rename_opts(from, to, options).await;
}
let from = PathBuf::from(from.as_ref());
let to = PathBuf::from(to.as_ref());
tokio::task::spawn_blocking(move || move_file_if_not_exists(&from, &to))
.await
.map_err(|source| Error::Generic {
store: STORE_NAME,
source: Box::new(source),
})?
}
}
fn move_file_if_not_exists(from: &StdPath, to: &StdPath) -> Result<()> {
let from_wide = null_terminated_wide(from.as_os_str());
let to_wide = null_terminated_wide(to.as_os_str());
// SAFETY: both pointers reference null-terminated UTF-16 buffers that remain
// alive for the duration of this call. A zero flag value deliberately omits
// MOVEFILE_REPLACE_EXISTING, giving this operation create-only semantics.
if unsafe { MoveFileExW(from_wide.as_ptr(), to_wide.as_ptr(), 0) } != 0 {
return Ok(());
}
let source = std::io::Error::last_os_error();
let path = to.to_string_lossy().into_owned();
match source.raw_os_error().map(|code| code as u32) {
Some(ERROR_ALREADY_EXISTS | ERROR_FILE_EXISTS) => Err(Error::AlreadyExists {
path,
source: Box::new(source),
}),
_ if source.kind() == std::io::ErrorKind::NotFound => Err(Error::NotFound {
path,
source: Box::new(source),
}),
_ => Err(Error::Generic {
store: STORE_NAME,
source: Box::new(source),
}),
}
}
fn null_terminated_wide(value: &OsStr) -> Vec<u16> {
value.encode_wide().chain(Some(0)).collect()
}
#[cfg(test)]
mod tests {
use object_store::memory::InMemory;
use super::*;
#[tokio::test]
async fn create_only_rename_does_not_use_hard_links() {
let tempdir = tempfile::tempdir().unwrap();
let source_path = tempdir.path().join("staged.manifest");
let destination_path = tempdir.path().join("1.manifest");
std::fs::write(&source_path, b"manifest").unwrap();
let source = Path::from_absolute_path(&source_path).unwrap();
let destination = Path::from_absolute_path(&destination_path).unwrap();
let store = WindowsLocalFileSystem {
// The source does not exist in this inner store. Delegating the
// rename would fail, proving the wrapper uses the native move path.
target: Arc::new(InMemory::new()),
};
store
.rename_opts(
&source,
&destination,
RenameOptions::new().with_target_mode(RenameTargetMode::Create),
)
.await
.unwrap();
assert!(!source_path.exists());
assert_eq!(std::fs::read(destination_path).unwrap(), b"manifest");
}
#[tokio::test]
async fn create_only_rename_preserves_existing_destination() {
let tempdir = tempfile::tempdir().unwrap();
let source_path = tempdir.path().join("staged.manifest");
let destination_path = tempdir.path().join("1.manifest");
std::fs::write(&source_path, b"new manifest").unwrap();
std::fs::write(&destination_path, b"existing manifest").unwrap();
let source = Path::from_absolute_path(&source_path).unwrap();
let destination = Path::from_absolute_path(&destination_path).unwrap();
let store = WindowsLocalFileSystem {
target: Arc::new(InMemory::new()),
};
let error = store
.rename_opts(
&source,
&destination,
RenameOptions::new().with_target_mode(RenameTargetMode::Create),
)
.await
.unwrap_err();
assert!(matches!(error, Error::AlreadyExists { .. }));
assert_eq!(std::fs::read(source_path).unwrap(), b"new manifest");
assert_eq!(
std::fs::read(destination_path).unwrap(),
b"existing manifest"
);
}
}
+10 -99
View File
@@ -2335,19 +2335,10 @@ impl NativeTable {
managed_versioning: Option<bool>,
) -> Result<Self> {
let params = params.unwrap_or_default();
let has_caller_store_wrapper = params
.store_options
.as_ref()
.and_then(|options| options.object_store_wrapper.as_ref())
.is_some();
// A caller wrapper must remain outside connection-level compatibility
// behavior. When there is no caller wrapper, apply the compatibility
// layer after loading so the session's registered store can be reused.
let (params, wrapper_after_load) = match write_store_wrapper {
Some(wrapper) if has_caller_store_wrapper => {
(params.patch_with_store_wrapper(wrapper)?, None)
}
wrapper => (params, wrapper),
// patch the params if we have a write store wrapper
let params = match write_store_wrapper.clone() {
Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
None => params,
};
// Build table_id from namespace + name
@@ -2406,14 +2397,6 @@ impl NativeTable {
}
Err(e) => return Err(e.into()),
};
// Resolve the store from the session registry before applying a
// connection-level write wrapper. Wrapper identity is part of the
// registry key, so including it in ReadParams prevents reuse when the
// opened table (and its wrapped store) is short-lived.
let dataset = match wrapper_after_load {
Some(wrapper) => dataset.with_object_store_wrappers([wrapper]),
None => dataset,
};
let dataset = DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval);
let id = Self::build_id(&namespace, name);
@@ -2515,16 +2498,11 @@ impl NativeTable {
if let Some(sess) = session {
params.session(sess);
}
let has_caller_store_wrapper = params
.store_options
.as_ref()
.and_then(|options| options.object_store_wrapper.as_ref())
.is_some();
let (params, wrapper_after_load) = match write_store_wrapper {
Some(wrapper) if has_caller_store_wrapper => {
(params.patch_with_store_wrapper(wrapper)?, None)
}
wrapper => (params, wrapper),
// patch the params if we have a write store wrapper
let params = match write_store_wrapper.clone() {
Some(wrapper) => params.patch_with_store_wrapper(wrapper)?,
None => params,
};
// Build table_id from namespace + name
@@ -2548,13 +2526,6 @@ impl NativeTable {
},
e => e.into(),
})?;
// Apply the write wrapper after the session registry has resolved the
// shared store. The cloned dataset retains the wrapper for subsequent
// reads, manifest commits, and any additional base stores.
let dataset = match wrapper_after_load {
Some(wrapper) => dataset.with_object_store_wrappers([wrapper]),
None => dataset,
};
let uri = dataset.uri().to_string();
let dataset = DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval);
@@ -3689,8 +3660,8 @@ pub struct FragmentSummaryStats {
#[cfg(test)]
#[allow(deprecated)]
mod tests {
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use arrow_array::{
@@ -4062,66 +4033,6 @@ mod tests {
}
}
#[derive(Debug)]
struct OrderedStoreWrapper {
name: &'static str,
order: Arc<Mutex<Vec<&'static str>>>,
}
impl WrappingObjectStore for OrderedStoreWrapper {
fn wrap(
&self,
_store_prefix: &str,
original: Arc<dyn object_store::ObjectStore>,
) -> Arc<dyn object_store::ObjectStore> {
self.order.lock().unwrap().push(self.name);
original
}
}
#[tokio::test]
async fn test_open_with_params_keeps_caller_store_wrapper_outermost() {
let tmp_dir = tempdir().unwrap();
let dataset_path = tmp_dir.path().join("test.lance");
let uri = dataset_path.to_str().unwrap();
let batch = make_test_batches();
let reader = RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
Dataset::write(reader, uri, None).await.unwrap();
let order = Arc::new(Mutex::new(Vec::new()));
let caller_wrapper = Arc::new(OrderedStoreWrapper {
name: "caller",
order: order.clone(),
});
let compatibility_wrapper = Arc::new(OrderedStoreWrapper {
name: "compatibility",
order: order.clone(),
});
let params = ReadParams {
store_options: Some(ObjectStoreParams {
object_store_wrapper: Some(caller_wrapper),
..Default::default()
}),
..Default::default()
};
NativeTable::open_with_params(
uri,
"test",
vec![],
Some(compatibility_wrapper),
Some(params),
None,
None,
HashSet::new(),
None,
)
.await
.unwrap();
assert_eq!(*order.lock().unwrap(), vec!["compatibility", "caller"]);
}
#[tokio::test]
async fn test_open_table_options() {
let tmp_dir = tempdir().unwrap();
+7 -54
View File
@@ -14,7 +14,6 @@ use lance::arrow::json::JsonDataType;
use lance::dataset::{ReadParams, WriteParams};
use lance::index::vector::utils::infer_vector_dim;
use lance::io::{ObjectStoreParams, WrappingObjectStore};
use lance_io::object_store::ChainedWrappingObjectStore;
use std::pin::Pin;
use crate::error::{Error, Result};
@@ -38,13 +37,13 @@ impl PatchStoreParam for Option<ObjectStoreParams> {
wrapper: Arc<dyn WrappingObjectStore>,
) -> Result<Option<ObjectStoreParams>> {
let mut params = self.unwrap_or_default();
params.object_store_wrapper = Some(match params.object_store_wrapper.take() {
// The wrapper being patched in is connection-level compatibility
// behavior. Keep it closest to the target store so an existing
// caller wrapper remains outermost and can observe every operation.
Some(existing) => Arc::new(ChainedWrappingObjectStore::new(vec![wrapper, existing])),
None => wrapper,
});
if params.object_store_wrapper.is_some() {
return Err(Error::Other {
message: "can not patch param because object store is already set".into(),
source: None,
});
}
params.object_store_wrapper = Some(wrapper);
Ok(Some(params))
}
@@ -473,60 +472,14 @@ impl Stream for MaxBatchLengthStream {
#[cfg(test)]
mod tests {
use std::sync::Mutex;
use arrow_array::Int32Array;
use arrow_schema::Field;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
use futures::{StreamExt, stream};
use object_store::{ObjectStore, memory::InMemory};
use tokio::time::sleep;
use super::*;
#[derive(Debug)]
struct OrderedStoreWrapper {
name: &'static str,
order: Arc<Mutex<Vec<&'static str>>>,
}
impl WrappingObjectStore for OrderedStoreWrapper {
fn wrap(
&self,
_store_prefix: &str,
original: Arc<dyn ObjectStore>,
) -> Arc<dyn ObjectStore> {
self.order.lock().unwrap().push(self.name);
original
}
}
#[test]
fn test_patch_store_param_keeps_caller_wrapper_outermost() {
let order = Arc::new(Mutex::new(Vec::new()));
let params = Some(ObjectStoreParams {
object_store_wrapper: Some(Arc::new(OrderedStoreWrapper {
name: "caller",
order: order.clone(),
})),
..Default::default()
});
let params = params
.patch_with_store_wrapper(Arc::new(OrderedStoreWrapper {
name: "compatibility",
order: order.clone(),
}))
.unwrap()
.unwrap();
params
.object_store_wrapper
.unwrap()
.wrap("memory", Arc::new(InMemory::new()) as Arc<dyn ObjectStore>);
assert_eq!(*order.lock().unwrap(), vec!["compatibility", "caller"]);
}
#[test]
fn test_guess_default_column() {
let schema_no_vector = Schema::new(vec![