diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml index a2842cb1a..b43f27354 100644 --- a/.github/workflows/build_linux_wheel/action.yml +++ b/.github/workflows/build_linux_wheel/action.yml @@ -34,15 +34,16 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc" target: x86_64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e - curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-$(uname -m).zip > /tmp/protoc.zip \ - && unzip /tmp/protoc.zip -d /usr/local \ - && rm /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip + unzip /tmp/protoc.zip -d /usr/local + rm /tmp/protoc.zip + /usr/local/bin/protoc --version - name: Build Arm Manylinux Wheel if: ${{ inputs.arm-build == 'true' }} uses: PyO3/maturin-action@v1 @@ -50,13 +51,14 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/'" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc" target: aarch64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e - yum install -y clang \ - && curl -L https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip > /tmp/protoc.zip \ - && unzip /tmp/protoc.zip -d /usr/local \ - && rm /tmp/protoc.zip + yum install -y clang + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip + unzip /tmp/protoc.zip -d /usr/local + rm /tmp/protoc.zip + /usr/local/bin/protoc --version diff --git a/Cargo.lock b/Cargo.lock index f8533441d..ed04aa938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3423,8 +3423,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "rand 0.9.4", @@ -4774,8 +4774,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arc-swap", "arrow", @@ -4849,8 +4849,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4872,7 +4872,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4886,7 +4886,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-schema", @@ -4895,8 +4895,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrayref", "crunchy", @@ -4906,8 +4906,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow", "arrow-array", @@ -4976,8 +4976,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow", "arrow-array", @@ -4994,8 +4994,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "proc-macro2", "quote", @@ -5004,8 +5004,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-arith", "arrow-array", @@ -5040,8 +5040,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-arith", "arrow-array", @@ -5071,8 +5071,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arc-swap", "arrow", @@ -5137,8 +5137,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow", "arrow-arith", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-buffer", @@ -5196,8 +5196,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow", "async-trait", @@ -5209,8 +5209,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow", "arrow-ipc", @@ -5233,7 +5233,7 @@ dependencies = [ "lance-table", "log", "object_store", - "quick-xml 0.38.4", + "quick-xml 0.40.1", "rand 0.9.4", "reqwest 0.12.28", "roaring", @@ -5264,8 +5264,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-buffer", @@ -5280,8 +5280,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow", "arrow-array", @@ -5320,8 +5320,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "arrow-array", "arrow-schema", @@ -5334,8 +5334,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "9.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd" +version = "9.0.0-beta.19" +source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e" dependencies = [ "icu_segmenter", "jieba-rs", @@ -7688,15 +7688,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "quick-xml" -version = "0.38.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.39.4" @@ -7707,6 +7698,15 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.9" diff --git a/Cargo.toml b/Cargo.toml index 3b5a5fef3..712a0b6c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" } ahash = "0.8" # Note that this one does not include pyarrow arrow = { version = "58.0.0", optional = false } diff --git a/java/pom.xml b/java/pom.xml index 4f8ae3983..041b75d3a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 9.0.0-beta.18 + 9.0.0-beta.19 false 2.30.0 1.7 diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 5fa700156..9a7f2c415 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -149,8 +149,14 @@ def connect( For object storage, use a URI prefix: - >>> db = lancedb.connect("s3://my-bucket/lancedb", - ... storage_options={"aws_access_key_id": "***"}) + >>> db = lancedb.connect( # doctest: +SKIP + ... "s3://my-bucket/lancedb", + ... storage_options={ + ... "aws_access_key_id": "***", + ... "aws_secret_access_key": "***", + ... "aws_region": "us-east-1", + ... }, + ... ) For tests and temporary data, use an in-memory database: diff --git a/python/python/tests/test_s3_bucket_dots.py b/python/python/tests/test_s3_bucket_dots.py index 5c087f808..146b8c3ad 100644 --- a/python/python/tests/test_s3_bucket_dots.py +++ b/python/python/tests/test_s3_bucket_dots.py @@ -7,7 +7,8 @@ Tests for S3 bucket names containing dots. Related issue: https://github.com/lancedb/lancedb/issues/1898 These tests validate the early error checking for S3 bucket names with dots. -No actual S3 connection is made - validation happens before connection. +When validation succeeds, eager namespace initialization may still fail later +because these tests intentionally do not provide real S3 credentials. """ import pytest @@ -20,6 +21,13 @@ BUCKET_WITH_DOTS_AND_AWS_REGION = ("s3://my.bucket.name", {"aws_region": "us-eas BUCKET_WITHOUT_DOTS = "s3://my-bucket/path" +def assert_not_rejected_for_bucket_dots(connect): + try: + connect() + except ValueError as err: + assert "contains dots" not in str(err) + + class TestS3BucketWithDotsSync: """Tests for connect().""" @@ -27,19 +35,22 @@ class TestS3BucketWithDotsSync: with pytest.raises(ValueError, match="contains dots"): lancedb.connect(BUCKET_WITH_DOTS) - def test_bucket_with_dots_and_region_passes(self): + def test_bucket_with_dots_and_region_is_not_rejected(self): uri, opts = BUCKET_WITH_DOTS_AND_REGION - db = lancedb.connect(uri, storage_options=opts) - assert db is not None + assert_not_rejected_for_bucket_dots( + lambda: lancedb.connect(uri, storage_options=opts) + ) - def test_bucket_with_dots_and_aws_region_passes(self): + def test_bucket_with_dots_and_aws_region_is_not_rejected(self): uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION - db = lancedb.connect(uri, storage_options=opts) - assert db is not None + assert_not_rejected_for_bucket_dots( + lambda: lancedb.connect(uri, storage_options=opts) + ) - def test_bucket_without_dots_passes(self): - db = lancedb.connect(BUCKET_WITHOUT_DOTS) - assert db is not None + def test_bucket_without_dots_is_not_rejected(self): + assert_not_rejected_for_bucket_dots( + lambda: lancedb.connect(BUCKET_WITHOUT_DOTS) + ) class TestS3BucketWithDotsAsync: @@ -51,18 +62,24 @@ class TestS3BucketWithDotsAsync: await lancedb.connect_async(BUCKET_WITH_DOTS) @pytest.mark.asyncio - async def test_bucket_with_dots_and_region_passes(self): + async def test_bucket_with_dots_and_region_is_not_rejected(self): uri, opts = BUCKET_WITH_DOTS_AND_REGION - db = await lancedb.connect_async(uri, storage_options=opts) - assert db is not None + try: + await lancedb.connect_async(uri, storage_options=opts) + except ValueError as err: + assert "contains dots" not in str(err) @pytest.mark.asyncio - async def test_bucket_with_dots_and_aws_region_passes(self): + async def test_bucket_with_dots_and_aws_region_is_not_rejected(self): uri, opts = BUCKET_WITH_DOTS_AND_AWS_REGION - db = await lancedb.connect_async(uri, storage_options=opts) - assert db is not None + try: + await lancedb.connect_async(uri, storage_options=opts) + except ValueError as err: + assert "contains dots" not in str(err) @pytest.mark.asyncio - async def test_bucket_without_dots_passes(self): - db = await lancedb.connect_async(BUCKET_WITHOUT_DOTS) - assert db is not None + async def test_bucket_without_dots_is_not_rejected(self): + try: + await lancedb.connect_async(BUCKET_WITHOUT_DOTS) + except ValueError as err: + assert "contains dots" not in str(err) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index f31274698..454498d54 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -342,6 +342,19 @@ impl ListingDatabase { )) } + fn storage_base_uri(uri: &str) -> String { + let Ok(mut url) = url::Url::parse(uri) else { + return uri.to_string(); + }; + url.set_query(None); + let Some((storage_scheme, _commit_scheme)) = url.scheme().split_once('+') else { + return url.to_string(); + }; + let storage_scheme = storage_scheme.to_string(); + let _ = url.set_scheme(&storage_scheme); + url.to_string() + } + async fn prepare_namespace_root( uri: &str, storage_options: &HashMap, @@ -520,6 +533,8 @@ impl ListingDatabase { // will add a trailing '?' to the url url.set_query(None); + let storage_base_uri = Self::storage_base_uri(url.as_str()); + let table_base_uri = if let Some(store) = engine { static WARN_ONCE: std::sync::Once = std::sync::Once::new(); WARN_ONCE.call_once(|| { @@ -532,8 +547,6 @@ impl ListingDatabase { url.to_string() }; - let plain_uri = url.to_string(); - let session = request .session .clone() @@ -550,13 +563,13 @@ impl ListingDatabase { }; let (object_store, base_path) = ObjectStore::from_uri_and_params( session.store_registry(), - &plain_uri, + &storage_base_uri, &os_params, ) .await?; if object_store.is_local() { - Self::try_create_dir(&plain_uri).context(CreateDirSnafu { - path: plain_uri.clone(), + Self::try_create_dir(&storage_base_uri).context(CreateDirSnafu { + path: storage_base_uri.clone(), })?; } @@ -570,7 +583,7 @@ impl ListingDatabase { }; let namespace_database = Self::connect_namespace_database( - &table_base_uri, + &storage_base_uri, options.storage_options.clone(), request.namespace_client_properties.clone(), request.read_consistency_interval, @@ -1309,6 +1322,60 @@ mod tests { (tempdir, db) } + #[tokio::test] + async fn test_listing_database_root_ops_do_not_create_manifest() { + let tempdir = tempdir().unwrap(); + let uri = tempdir.path().to_str().unwrap(); + + let request = ConnectRequest { + uri: uri.to_string(), + #[cfg(feature = "remote")] + client_config: Default::default(), + options: Default::default(), + namespace_client_properties: Default::default(), + manifest_enabled: false, + read_consistency_interval: None, + session: None, + }; + + let db = ListingDatabase::connect_with_options(&request) + .await + .unwrap(); + + assert!(!tempdir.path().join("__manifest").exists()); + + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + db.create_table(CreateTableRequest { + name: "root_table".to_string(), + namespace_path: vec![], + data: Box::new(RecordBatch::new_empty(schema)) as Box, + mode: CreateTableMode::Create, + write_options: Default::default(), + location: None, + namespace_client: None, + }) + .await + .unwrap(); + + db.open_table(OpenTableRequest { + name: "root_table".to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + .unwrap(); + + #[allow(deprecated)] + let table_names = db.table_names(TableNamesRequest::default()).await.unwrap(); + + assert_eq!(table_names, vec!["root_table".to_string()]); + assert!(!tempdir.path().join("__manifest").exists()); + } + #[tokio::test] async fn test_clone_table_basic() { let (_tempdir, db) = setup_database().await; @@ -2283,6 +2350,24 @@ mod tests { assert_eq!(captured.as_deref(), Some("foo=bar")); } + #[test] + fn test_storage_base_uri_strips_commit_engine_scheme() { + assert_eq!( + ListingDatabase::storage_base_uri("s3+ddb://bucket/prefix?ddbTableName=commit_table"), + "s3://bucket/prefix" + ); + + assert_eq!( + ListingDatabase::storage_base_uri("s3://bucket/prefix?foo=bar"), + "s3://bucket/prefix" + ); + + assert_eq!( + ListingDatabase::storage_base_uri("/tmp/lancedb"), + "/tmp/lancedb" + ); + } + /// Regression: connecting via a URL-style URI (which goes through /// `url::Url::parse` and the `query_pairs_mut()` path) must not /// append a trailing `?` to per-table URIs when the input URI has