mirror of
https://github.com/lancedb/lancedb.git
synced 2026-07-10 14:30:41 +00:00
fix: avoid manifest writes for read-only directory namespace opens (#3635)
Bumps Lance to v9.0.0-beta.19, which includes lance-format/lance#7687 for side-effect-free DirectoryNamespace read paths. This fixes root-level read-only table opens that previously could trigger `__manifest` creation through directory namespace construction, including Hugging Face bucket reads with read-only tokens. A LanceDB regression test now covers root listing operations without creating `__manifest`. Fixes #3633.
This commit is contained in:
Xuanwo
GitHub
20
.github/workflows/build_linux_wheel/action.yml
vendored
20
.github/workflows/build_linux_wheel/action.yml
vendored
@@ -34,15 +34,16 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
|
||||
target: x86_64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \
|
||||
&& unzip /tmp/protoc.zip -d /usr/local \
|
||||
&& rm /tmp/protoc.zip
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
- name: Build Arm Manylinux Wheel
|
||||
if: ${{ inputs.arm-build == 'true' }}
|
||||
uses: PyO3/maturin-action@v1
|
||||
@@ -50,13 +51,14 @@ runs:
|
||||
maturin-version: "1.12.4"
|
||||
command: build
|
||||
working-directory: python
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'"
|
||||
docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc"
|
||||
target: aarch64-unknown-linux-gnu
|
||||
manylinux: ${{ inputs.manylinux }}
|
||||
args: ${{ inputs.args }}
|
||||
before-script-linux: |
|
||||
set -e
|
||||
yum install -y clang \
|
||||
&& curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \
|
||||
&& unzip /tmp/protoc.zip -d /usr/local \
|
||||
&& rm /tmp/protoc.zip
|
||||
yum install -y clang
|
||||
curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip
|
||||
unzip /tmp/protoc.zip -d /usr/local
|
||||
rm /tmp/protoc.zip
|
||||
/usr/local/bin/protoc --version
|
||||
|
||||
100
Cargo.lock
generated
100
Cargo.lock
generated
@@ -3423,8 +3423,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.4",
|
||||
@@ -4774,8 +4774,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4849,8 +4849,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4872,7 +4872,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4886,7 +4886,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4895,8 +4895,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4906,8 +4906,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4976,8 +4976,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4994,8 +4994,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5004,8 +5004,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5040,8 +5040,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5071,8 +5071,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5137,8 +5137,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -5179,8 +5179,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5196,8 +5196,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5209,8 +5209,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5233,7 +5233,7 @@ dependencies = [
|
||||
"lance-table",
|
||||
"log",
|
||||
"object_store",
|
||||
"quick-xml 0.38.4",
|
||||
"quick-xml 0.40.1",
|
||||
"rand 0.9.4",
|
||||
"reqwest 0.12.28",
|
||||
"roaring",
|
||||
@@ -5264,8 +5264,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5280,8 +5280,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5320,8 +5320,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5334,8 +5334,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "9.0.0-beta.18"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
|
||||
version = "9.0.0-beta.19"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
|
||||
dependencies = [
|
||||
"icu_segmenter",
|
||||
"jieba-rs",
|
||||
@@ -7688,15 +7688,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.39.4"
|
||||
@@ -7707,6 +7698,15 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.40.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
|
||||
28
Cargo.toml
28
Cargo.toml
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>9.0.0-beta.18</lance-core.version>
|
||||
<lance-core.version>9.0.0-beta.19</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -149,8 +149,14 @@ def connect(
|
||||
|
||||
For object storage, use a URI prefix:
|
||||
|
||||
>>> db = lancedb.connect("s3://my-bucket/lancedb",
|
||||
... storage_options={"aws_access_key_id": "***"})
|
||||
>>> db = lancedb.connect( # doctest: +SKIP
|
||||
... "s3://my-bucket/lancedb",
|
||||
... storage_options={
|
||||
... "aws_access_key_id": "***",
|
||||
... "aws_secret_access_key": "***",
|
||||
... "aws_region": "us-east-1",
|
||||
... },
|
||||
... )
|
||||
|
||||
For tests and temporary data, use an in-memory database:
|
||||
|
||||
|
||||
@@ -7,7 +7,8 @@ Tests for S3 bucket names containing dots.
|
||||
Related issue: https://github.com/lancedb/lancedb/issues/1898
|
||||
|
||||
These tests validate the early error checking for S3 bucket names with dots.
|
||||
No actual S3 connection is made - validation happens before connection.
|
||||
When validation succeeds, eager namespace initialization may still fail later
|
||||
because these tests intentionally do not provide real S3 credentials.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -20,6 +21,13 @@ BUCKET_WITH_DOTS_AND_AWS_REGION = ("s3://my.bucket.name", {"aws_region": "us-eas
|
||||
BUCKET_WITHOUT_DOTS = "s3://my-bucket/path"
|
||||
|
||||
|
||||
def assert_not_rejected_for_bucket_dots(connect):
|
||||
try:
|
||||
connect()
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
|
||||
|
||||
class TestS3BucketWithDotsSync:
|
||||
"""Tests for connect()."""
|
||||
|
||||
@@ -27,19 +35,22 @@ class TestS3BucketWithDotsSync:
|
||||
with pytest.raises(ValueError, match="contains dots"):
|
||||
lancedb.connect(BUCKET_WITH_DOTS)
|
||||
|
||||
def test_bucket_with_dots_and_region_passes(self):
|
||||
def test_bucket_with_dots_and_region_is_not_rejected(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_REGION
|
||||
db = lancedb.connect(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
assert_not_rejected_for_bucket_dots(
|
||||
lambda: lancedb.connect(uri, storage_options=opts)
|
||||
)
|
||||
|
||||
def test_bucket_with_dots_and_aws_region_passes(self):
|
||||
def test_bucket_with_dots_and_aws_region_is_not_rejected(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
|
||||
db = lancedb.connect(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
assert_not_rejected_for_bucket_dots(
|
||||
lambda: lancedb.connect(uri, storage_options=opts)
|
||||
)
|
||||
|
||||
def test_bucket_without_dots_passes(self):
|
||||
db = lancedb.connect(BUCKET_WITHOUT_DOTS)
|
||||
assert db is not None
|
||||
def test_bucket_without_dots_is_not_rejected(self):
|
||||
assert_not_rejected_for_bucket_dots(
|
||||
lambda: lancedb.connect(BUCKET_WITHOUT_DOTS)
|
||||
)
|
||||
|
||||
|
||||
class TestS3BucketWithDotsAsync:
|
||||
@@ -51,18 +62,24 @@ class TestS3BucketWithDotsAsync:
|
||||
await lancedb.connect_async(BUCKET_WITH_DOTS)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bucket_with_dots_and_region_passes(self):
|
||||
async def test_bucket_with_dots_and_region_is_not_rejected(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_REGION
|
||||
db = await lancedb.connect_async(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
try:
|
||||
await lancedb.connect_async(uri, storage_options=opts)
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bucket_with_dots_and_aws_region_passes(self):
|
||||
async def test_bucket_with_dots_and_aws_region_is_not_rejected(self):
|
||||
uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION
|
||||
db = await lancedb.connect_async(uri, storage_options=opts)
|
||||
assert db is not None
|
||||
try:
|
||||
await lancedb.connect_async(uri, storage_options=opts)
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bucket_without_dots_passes(self):
|
||||
db = await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
|
||||
assert db is not None
|
||||
async def test_bucket_without_dots_is_not_rejected(self):
|
||||
try:
|
||||
await lancedb.connect_async(BUCKET_WITHOUT_DOTS)
|
||||
except ValueError as err:
|
||||
assert "contains dots" not in str(err)
|
||||
|
||||
@@ -342,6 +342,19 @@ impl ListingDatabase {
|
||||
))
|
||||
}
|
||||
|
||||
fn storage_base_uri(uri: &str) -> String {
|
||||
let Ok(mut url) = url::Url::parse(uri) else {
|
||||
return uri.to_string();
|
||||
};
|
||||
url.set_query(None);
|
||||
let Some((storage_scheme, _commit_scheme)) = url.scheme().split_once('+') else {
|
||||
return url.to_string();
|
||||
};
|
||||
let storage_scheme = storage_scheme.to_string();
|
||||
let _ = url.set_scheme(&storage_scheme);
|
||||
url.to_string()
|
||||
}
|
||||
|
||||
async fn prepare_namespace_root(
|
||||
uri: &str,
|
||||
storage_options: &HashMap<String, String>,
|
||||
@@ -520,6 +533,8 @@ impl ListingDatabase {
|
||||
// will add a trailing '?' to the url
|
||||
url.set_query(None);
|
||||
|
||||
let storage_base_uri = Self::storage_base_uri(url.as_str());
|
||||
|
||||
let table_base_uri = if let Some(store) = engine {
|
||||
static WARN_ONCE: std::sync::Once = std::sync::Once::new();
|
||||
WARN_ONCE.call_once(|| {
|
||||
@@ -532,8 +547,6 @@ impl ListingDatabase {
|
||||
url.to_string()
|
||||
};
|
||||
|
||||
let plain_uri = url.to_string();
|
||||
|
||||
let session = request
|
||||
.session
|
||||
.clone()
|
||||
@@ -550,13 +563,13 @@ impl ListingDatabase {
|
||||
};
|
||||
let (object_store, base_path) = ObjectStore::from_uri_and_params(
|
||||
session.store_registry(),
|
||||
&plain_uri,
|
||||
&storage_base_uri,
|
||||
&os_params,
|
||||
)
|
||||
.await?;
|
||||
if object_store.is_local() {
|
||||
Self::try_create_dir(&plain_uri).context(CreateDirSnafu {
|
||||
path: plain_uri.clone(),
|
||||
Self::try_create_dir(&storage_base_uri).context(CreateDirSnafu {
|
||||
path: storage_base_uri.clone(),
|
||||
})?;
|
||||
}
|
||||
|
||||
@@ -570,7 +583,7 @@ impl ListingDatabase {
|
||||
};
|
||||
|
||||
let namespace_database = Self::connect_namespace_database(
|
||||
&table_base_uri,
|
||||
&storage_base_uri,
|
||||
options.storage_options.clone(),
|
||||
request.namespace_client_properties.clone(),
|
||||
request.read_consistency_interval,
|
||||
@@ -1309,6 +1322,60 @@ mod tests {
|
||||
(tempdir, db)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_listing_database_root_ops_do_not_create_manifest() {
|
||||
let tempdir = tempdir().unwrap();
|
||||
let uri = tempdir.path().to_str().unwrap();
|
||||
|
||||
let request = ConnectRequest {
|
||||
uri: uri.to_string(),
|
||||
#[cfg(feature = "remote")]
|
||||
client_config: Default::default(),
|
||||
options: Default::default(),
|
||||
namespace_client_properties: Default::default(),
|
||||
manifest_enabled: false,
|
||||
read_consistency_interval: None,
|
||||
session: None,
|
||||
};
|
||||
|
||||
let db = ListingDatabase::connect_with_options(&request)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!tempdir.path().join("__manifest").exists());
|
||||
|
||||
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
|
||||
db.create_table(CreateTableRequest {
|
||||
name: "root_table".to_string(),
|
||||
namespace_path: vec![],
|
||||
data: Box::new(RecordBatch::new_empty(schema)) as Box<dyn Scannable>,
|
||||
mode: CreateTableMode::Create,
|
||||
write_options: Default::default(),
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
db.open_table(OpenTableRequest {
|
||||
name: "root_table".to_string(),
|
||||
namespace_path: vec![],
|
||||
index_cache_size: None,
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
#[allow(deprecated)]
|
||||
let table_names = db.table_names(TableNamesRequest::default()).await.unwrap();
|
||||
|
||||
assert_eq!(table_names, vec!["root_table".to_string()]);
|
||||
assert!(!tempdir.path().join("__manifest").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_clone_table_basic() {
|
||||
let (_tempdir, db) = setup_database().await;
|
||||
@@ -2283,6 +2350,24 @@ mod tests {
|
||||
assert_eq!(captured.as_deref(), Some("foo=bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_base_uri_strips_commit_engine_scheme() {
|
||||
assert_eq!(
|
||||
ListingDatabase::storage_base_uri("s3+ddb://bucket/prefix?ddbTableName=commit_table"),
|
||||
"s3://bucket/prefix"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
ListingDatabase::storage_base_uri("s3://bucket/prefix?foo=bar"),
|
||||
"s3://bucket/prefix"
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
ListingDatabase::storage_base_uri("/tmp/lancedb"),
|
||||
"/tmp/lancedb"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression: connecting via a URL-style URI (which goes through
|
||||
/// `url::Url::parse` and the `query_pairs_mut()` path) must not
|
||||
/// append a trailing `?` to per-table URIs when the input URI has
|
||||
|
||||
Reference in New Issue
Block a user