mirror of
https://github.com/lancedb/lancedb.git
synced 2026-03-28 19:40:39 +00:00
Compare commits
1 Commits
python-v0.
...
codex/upda
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3e4be3c4b5 |
116
Cargo.lock
generated
116
Cargo.lock
generated
@@ -3088,8 +3088,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.2",
|
||||
@@ -4260,8 +4260,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4315,7 +4315,7 @@ dependencies = [
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tantivy",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -4327,8 +4327,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4338,6 +4338,7 @@ dependencies = [
|
||||
"arrow-schema",
|
||||
"arrow-select",
|
||||
"bytes",
|
||||
"futures",
|
||||
"getrandom 0.2.16",
|
||||
"half",
|
||||
"jsonb",
|
||||
@@ -4347,8 +4348,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"paste",
|
||||
@@ -4357,8 +4358,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4384,7 +4385,7 @@ dependencies = [
|
||||
"rand 0.9.2",
|
||||
"roaring",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -4395,8 +4396,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4419,15 +4420,15 @@ dependencies = [
|
||||
"pin-project",
|
||||
"prost",
|
||||
"prost-build",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4445,8 +4446,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4473,7 +4474,7 @@ dependencies = [
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"rand 0.9.2",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"strum",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -4483,8 +4484,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -4509,15 +4510,15 @@ dependencies = [
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4569,7 +4570,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tantivy",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
@@ -4580,8 +4581,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-arith",
|
||||
@@ -4613,7 +4614,7 @@ dependencies = [
|
||||
"prost",
|
||||
"rand 0.9.2",
|
||||
"serde",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -4622,8 +4623,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4639,21 +4640,21 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"lance-core",
|
||||
"lance-namespace-reqwest-client",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -4675,7 +4676,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tower-http 0.5.2",
|
||||
@@ -4697,8 +4698,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -4728,7 +4729,7 @@ dependencies = [
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"snafu 0.9.0",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
@@ -4737,8 +4738,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "3.0.0-rc.2"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v3.0.0-rc.2#3fb3e705b8a25ab1bb0fc9e1e0158e8a13356181"
|
||||
version = "4.0.0-beta.7"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v4.0.0-beta.7#e1e5689e9c1e58df75cfdc64e8f029004e98ec71"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4819,7 +4820,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_with",
|
||||
"snafu",
|
||||
"snafu 0.8.9",
|
||||
"tempfile",
|
||||
"test-log",
|
||||
"tokenizers",
|
||||
@@ -4855,21 +4856,17 @@ version = "0.30.0-beta.3"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"env_logger",
|
||||
"futures",
|
||||
"lance-core",
|
||||
"lance-io",
|
||||
"lance-namespace",
|
||||
"lance-namespace-impls",
|
||||
"lancedb",
|
||||
"pin-project",
|
||||
"pyo3",
|
||||
"pyo3-async-runtimes",
|
||||
"pyo3-build-config",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"snafu",
|
||||
"snafu 0.8.9",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
@@ -7781,7 +7778,16 @@ version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2"
|
||||
dependencies = [
|
||||
"snafu-derive",
|
||||
"snafu-derive 0.8.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6"
|
||||
dependencies = [
|
||||
"snafu-derive 0.9.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7796,6 +7802,18 @@ dependencies = [
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snafu-derive"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
|
||||
dependencies = [
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.114",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.5.10"
|
||||
|
||||
28
Cargo.toml
28
Cargo.toml
@@ -15,20 +15,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=3.0.0-rc.2", default-features = false, "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=3.0.0-rc.2", "tag" = "v3.0.0-rc.2", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=4.0.0-beta.7", default-features = false, "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=4.0.0-beta.7", "tag" = "v4.0.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "57.2", optional = false }
|
||||
|
||||
@@ -52,21 +52,14 @@ plugins:
|
||||
options:
|
||||
docstring_style: numpy
|
||||
heading_level: 3
|
||||
show_source: true
|
||||
show_symbol_type_in_heading: true
|
||||
show_signature_annotations: true
|
||||
show_root_heading: true
|
||||
show_docstring_examples: true
|
||||
show_docstring_attributes: false
|
||||
show_docstring_other_parameters: true
|
||||
show_symbol_type_heading: true
|
||||
show_labels: false
|
||||
show_if_no_docstring: true
|
||||
show_source: false
|
||||
members_order: source
|
||||
docstring_section_style: list
|
||||
signature_crossrefs: true
|
||||
separate_signature: true
|
||||
filters:
|
||||
- "!^_"
|
||||
import:
|
||||
# for cross references
|
||||
- https://arrow.apache.org/docs/objects.inv
|
||||
@@ -120,7 +113,7 @@ markdown_extensions:
|
||||
emoji_index: !!python/name:material.extensions.emoji.twemoji
|
||||
emoji_generator: !!python/name:material.extensions.emoji.to_svg
|
||||
- markdown.extensions.toc:
|
||||
toc_depth: 4
|
||||
toc_depth: 3
|
||||
permalink: true
|
||||
permalink_title: Anchor link to this section
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>3.1.0-beta.2</lance-core.version>
|
||||
<lance-core.version>4.0.0-beta.7</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
[tool.bumpversion]
|
||||
current_version = "0.30.0-beta.4"
|
||||
current_version = "0.30.0-beta.3"
|
||||
parse = """(?x)
|
||||
(?P<major>0|[1-9]\\d*)\\.
|
||||
(?P<minor>0|[1-9]\\d*)\\.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "lancedb-python"
|
||||
version = "0.30.0-beta.4"
|
||||
version = "0.30.0-beta.3"
|
||||
edition.workspace = true
|
||||
description = "Python bindings for LanceDB"
|
||||
license.workspace = true
|
||||
@@ -16,11 +16,9 @@ crate-type = ["cdylib"]
|
||||
[dependencies]
|
||||
arrow = { version = "57.2", features = ["pyarrow"] }
|
||||
async-trait = "0.1"
|
||||
bytes = "1"
|
||||
lancedb = { path = "../rust/lancedb", default-features = false }
|
||||
lance-core.workspace = true
|
||||
lance-namespace.workspace = true
|
||||
lance-namespace-impls.workspace = true
|
||||
lance-io.workspace = true
|
||||
env_logger.workspace = true
|
||||
pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
|
||||
@@ -30,8 +28,6 @@ pyo3-async-runtimes = { version = "0.26", features = [
|
||||
] }
|
||||
pin-project = "1.1.5"
|
||||
futures.workspace = true
|
||||
serde = "1"
|
||||
serde_json = "1"
|
||||
snafu.workspace = true
|
||||
tokio = { version = "1.40", features = ["sync"] }
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ repository = "https://github.com/lancedb/lancedb"
|
||||
|
||||
[project.optional-dependencies]
|
||||
pylance = [
|
||||
"pylance>=4.0.0b7",
|
||||
"pylance>=1.0.0b14",
|
||||
]
|
||||
tests = [
|
||||
"aiohttp",
|
||||
@@ -59,9 +59,9 @@ tests = [
|
||||
"polars>=0.19, <=1.3.0",
|
||||
"tantivy",
|
||||
"pyarrow-stubs",
|
||||
"pylance>=4.0.0b7",
|
||||
"pylance>=1.0.0b14,<3.0.0",
|
||||
"requests",
|
||||
"datafusion>=52,<53",
|
||||
"datafusion<52",
|
||||
]
|
||||
dev = [
|
||||
"ruff",
|
||||
|
||||
@@ -8,7 +8,7 @@ from abc import abstractmethod
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Literal, Optional, Union
|
||||
from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Union
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
@@ -1541,8 +1541,6 @@ class AsyncConnection(object):
|
||||
storage_options_provider: Optional["StorageOptionsProvider"] = None,
|
||||
index_cache_size: Optional[int] = None,
|
||||
location: Optional[str] = None,
|
||||
namespace_client: Optional[Any] = None,
|
||||
managed_versioning: Optional[bool] = None,
|
||||
) -> AsyncTable:
|
||||
"""Open a Lance Table in the database.
|
||||
|
||||
@@ -1575,9 +1573,6 @@ class AsyncConnection(object):
|
||||
The explicit location (URI) of the table. If provided, the table will be
|
||||
opened from this location instead of deriving it from the database URI
|
||||
and table name.
|
||||
managed_versioning: bool, optional
|
||||
Whether managed versioning is enabled for this table. If provided,
|
||||
avoids a redundant describe_table call when namespace_client is set.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -1592,8 +1587,6 @@ class AsyncConnection(object):
|
||||
storage_options_provider=storage_options_provider,
|
||||
index_cache_size=index_cache_size,
|
||||
location=location,
|
||||
namespace_client=namespace_client,
|
||||
managed_versioning=managed_versioning,
|
||||
)
|
||||
return AsyncTable(table)
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||
from typing import Dict, Iterable, List, Optional, Union
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override
|
||||
@@ -240,7 +240,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
session : Optional[Session]
|
||||
A session to use for this connection
|
||||
"""
|
||||
self._namespace_client = namespace
|
||||
self._ns = namespace
|
||||
self.read_consistency_interval = read_consistency_interval
|
||||
self.storage_options = storage_options or {}
|
||||
self.session = session
|
||||
@@ -269,7 +269,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
if namespace is None:
|
||||
namespace = []
|
||||
request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
|
||||
response = self._namespace_client.list_tables(request)
|
||||
response = self._ns.list_tables(request)
|
||||
return response.tables if response.tables else []
|
||||
|
||||
@override
|
||||
@@ -309,9 +309,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
# Try to describe the table first to see if it exists
|
||||
try:
|
||||
describe_request = DescribeTableRequest(id=table_id)
|
||||
describe_response = self._namespace_client.describe_table(
|
||||
describe_request
|
||||
)
|
||||
describe_response = self._ns.describe_table(describe_request)
|
||||
location = describe_response.location
|
||||
namespace_storage_options = describe_response.storage_options
|
||||
except Exception:
|
||||
@@ -325,7 +323,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
location=None,
|
||||
properties=self.storage_options if self.storage_options else None,
|
||||
)
|
||||
declare_response = self._namespace_client.declare_table(declare_request)
|
||||
declare_response = self._ns.declare_table(declare_request)
|
||||
|
||||
if not declare_response.location:
|
||||
raise ValueError(
|
||||
@@ -355,7 +353,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
# Only create if namespace returned storage_options (not None)
|
||||
if storage_options_provider is None and namespace_storage_options is not None:
|
||||
storage_options_provider = LanceNamespaceStorageOptionsProvider(
|
||||
namespace=self._namespace_client,
|
||||
namespace=self._ns,
|
||||
table_id=table_id,
|
||||
)
|
||||
|
||||
@@ -373,7 +371,6 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
storage_options=merged_storage_options,
|
||||
storage_options_provider=storage_options_provider,
|
||||
location=location,
|
||||
namespace_client=self._namespace_client,
|
||||
)
|
||||
|
||||
return tbl
|
||||
@@ -392,7 +389,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
namespace = []
|
||||
table_id = namespace + [name]
|
||||
request = DescribeTableRequest(id=table_id)
|
||||
response = self._namespace_client.describe_table(request)
|
||||
response = self._ns.describe_table(request)
|
||||
|
||||
# Merge storage options: self.storage_options < user options < namespace options
|
||||
merged_storage_options = dict(self.storage_options)
|
||||
@@ -405,14 +402,10 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
# Only create if namespace returned storage_options (not None)
|
||||
if storage_options_provider is None and response.storage_options is not None:
|
||||
storage_options_provider = LanceNamespaceStorageOptionsProvider(
|
||||
namespace=self._namespace_client,
|
||||
namespace=self._ns,
|
||||
table_id=table_id,
|
||||
)
|
||||
|
||||
# Pass managed_versioning to avoid redundant describe_table call in Rust.
|
||||
# Convert None to False since we already have the answer from describe_table.
|
||||
managed_versioning = response.managed_versioning is True
|
||||
|
||||
return self._lance_table_from_uri(
|
||||
name,
|
||||
response.location,
|
||||
@@ -420,8 +413,6 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
storage_options=merged_storage_options,
|
||||
storage_options_provider=storage_options_provider,
|
||||
index_cache_size=index_cache_size,
|
||||
namespace_client=self._namespace_client,
|
||||
managed_versioning=managed_versioning,
|
||||
)
|
||||
|
||||
@override
|
||||
@@ -431,7 +422,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
namespace = []
|
||||
table_id = namespace + [name]
|
||||
request = DropTableRequest(id=table_id)
|
||||
self._namespace_client.drop_table(request)
|
||||
self._ns.drop_table(request)
|
||||
|
||||
@override
|
||||
def rename_table(
|
||||
@@ -493,7 +484,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
request = ListNamespacesRequest(
|
||||
id=namespace, page_token=page_token, limit=limit
|
||||
)
|
||||
response = self._namespace_client.list_namespaces(request)
|
||||
response = self._ns.list_namespaces(request)
|
||||
return ListNamespacesResponse(
|
||||
namespaces=response.namespaces if response.namespaces else [],
|
||||
page_token=response.page_token,
|
||||
@@ -529,7 +520,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
mode=_normalize_create_namespace_mode(mode),
|
||||
properties=properties,
|
||||
)
|
||||
response = self._namespace_client.create_namespace(request)
|
||||
response = self._ns.create_namespace(request)
|
||||
return CreateNamespaceResponse(
|
||||
properties=response.properties if hasattr(response, "properties") else None
|
||||
)
|
||||
@@ -564,7 +555,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
mode=_normalize_drop_namespace_mode(mode),
|
||||
behavior=_normalize_drop_namespace_behavior(behavior),
|
||||
)
|
||||
response = self._namespace_client.drop_namespace(request)
|
||||
response = self._ns.drop_namespace(request)
|
||||
return DropNamespaceResponse(
|
||||
properties=(
|
||||
response.properties if hasattr(response, "properties") else None
|
||||
@@ -590,7 +581,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
Response containing the namespace properties.
|
||||
"""
|
||||
request = DescribeNamespaceRequest(id=namespace)
|
||||
response = self._namespace_client.describe_namespace(request)
|
||||
response = self._ns.describe_namespace(request)
|
||||
return DescribeNamespaceResponse(
|
||||
properties=response.properties if hasattr(response, "properties") else None
|
||||
)
|
||||
@@ -624,7 +615,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
if namespace is None:
|
||||
namespace = []
|
||||
request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
|
||||
response = self._namespace_client.list_tables(request)
|
||||
response = self._ns.list_tables(request)
|
||||
return ListTablesResponse(
|
||||
tables=response.tables if response.tables else [],
|
||||
page_token=response.page_token,
|
||||
@@ -639,8 +630,6 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
storage_options: Optional[Dict[str, str]] = None,
|
||||
storage_options_provider: Optional[StorageOptionsProvider] = None,
|
||||
index_cache_size: Optional[int] = None,
|
||||
namespace_client: Optional[Any] = None,
|
||||
managed_versioning: Optional[bool] = None,
|
||||
) -> LanceTable:
|
||||
# Open a table directly from a URI using the location parameter
|
||||
# Note: storage_options should already be merged by the caller
|
||||
@@ -654,8 +643,6 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
)
|
||||
|
||||
# Open the table using the temporary connection with the location parameter
|
||||
# Pass namespace_client to enable managed versioning support
|
||||
# Pass managed_versioning to avoid redundant describe_table call
|
||||
return LanceTable.open(
|
||||
temp_conn,
|
||||
name,
|
||||
@@ -664,8 +651,6 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
storage_options_provider=storage_options_provider,
|
||||
index_cache_size=index_cache_size,
|
||||
location=table_uri,
|
||||
namespace_client=namespace_client,
|
||||
managed_versioning=managed_versioning,
|
||||
)
|
||||
|
||||
|
||||
@@ -700,7 +685,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
session : Optional[Session]
|
||||
A session to use for this connection
|
||||
"""
|
||||
self._namespace_client = namespace
|
||||
self._ns = namespace
|
||||
self.read_consistency_interval = read_consistency_interval
|
||||
self.storage_options = storage_options or {}
|
||||
self.session = session
|
||||
@@ -728,7 +713,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
if namespace is None:
|
||||
namespace = []
|
||||
request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
|
||||
response = self._namespace_client.list_tables(request)
|
||||
response = self._ns.list_tables(request)
|
||||
return response.tables if response.tables else []
|
||||
|
||||
async def create_table(
|
||||
@@ -765,9 +750,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
# Try to describe the table first to see if it exists
|
||||
try:
|
||||
describe_request = DescribeTableRequest(id=table_id)
|
||||
describe_response = self._namespace_client.describe_table(
|
||||
describe_request
|
||||
)
|
||||
describe_response = self._ns.describe_table(describe_request)
|
||||
location = describe_response.location
|
||||
namespace_storage_options = describe_response.storage_options
|
||||
except Exception:
|
||||
@@ -781,7 +764,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
location=None,
|
||||
properties=self.storage_options if self.storage_options else None,
|
||||
)
|
||||
declare_response = self._namespace_client.declare_table(declare_request)
|
||||
declare_response = self._ns.declare_table(declare_request)
|
||||
|
||||
if not declare_response.location:
|
||||
raise ValueError(
|
||||
@@ -814,7 +797,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
and namespace_storage_options is not None
|
||||
):
|
||||
provider = LanceNamespaceStorageOptionsProvider(
|
||||
namespace=self._namespace_client,
|
||||
namespace=self._ns,
|
||||
table_id=table_id,
|
||||
)
|
||||
else:
|
||||
@@ -834,7 +817,6 @@ class AsyncLanceNamespaceDBConnection:
|
||||
storage_options=merged_storage_options,
|
||||
storage_options_provider=provider,
|
||||
location=location,
|
||||
namespace_client=self._namespace_client,
|
||||
)
|
||||
|
||||
lance_table = await asyncio.to_thread(_create_table)
|
||||
@@ -855,7 +837,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
namespace = []
|
||||
table_id = namespace + [name]
|
||||
request = DescribeTableRequest(id=table_id)
|
||||
response = self._namespace_client.describe_table(request)
|
||||
response = self._ns.describe_table(request)
|
||||
|
||||
# Merge storage options: self.storage_options < user options < namespace options
|
||||
merged_storage_options = dict(self.storage_options)
|
||||
@@ -867,14 +849,10 @@ class AsyncLanceNamespaceDBConnection:
|
||||
# Create a storage options provider if not provided by user
|
||||
if storage_options_provider is None and response.storage_options is not None:
|
||||
storage_options_provider = LanceNamespaceStorageOptionsProvider(
|
||||
namespace=self._namespace_client,
|
||||
namespace=self._ns,
|
||||
table_id=table_id,
|
||||
)
|
||||
|
||||
# Capture managed_versioning from describe response.
|
||||
# Convert None to False since we already have the answer from describe_table.
|
||||
managed_versioning = response.managed_versioning is True
|
||||
|
||||
# Open table in a thread
|
||||
def _open_table():
|
||||
temp_conn = LanceDBConnection(
|
||||
@@ -892,8 +870,6 @@ class AsyncLanceNamespaceDBConnection:
|
||||
storage_options_provider=storage_options_provider,
|
||||
index_cache_size=index_cache_size,
|
||||
location=response.location,
|
||||
namespace_client=self._namespace_client,
|
||||
managed_versioning=managed_versioning,
|
||||
)
|
||||
|
||||
lance_table = await asyncio.to_thread(_open_table)
|
||||
@@ -905,7 +881,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
namespace = []
|
||||
table_id = namespace + [name]
|
||||
request = DropTableRequest(id=table_id)
|
||||
self._namespace_client.drop_table(request)
|
||||
self._ns.drop_table(request)
|
||||
|
||||
async def rename_table(
|
||||
self,
|
||||
@@ -967,7 +943,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
request = ListNamespacesRequest(
|
||||
id=namespace, page_token=page_token, limit=limit
|
||||
)
|
||||
response = self._namespace_client.list_namespaces(request)
|
||||
response = self._ns.list_namespaces(request)
|
||||
return ListNamespacesResponse(
|
||||
namespaces=response.namespaces if response.namespaces else [],
|
||||
page_token=response.page_token,
|
||||
@@ -1002,7 +978,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
mode=_normalize_create_namespace_mode(mode),
|
||||
properties=properties,
|
||||
)
|
||||
response = self._namespace_client.create_namespace(request)
|
||||
response = self._ns.create_namespace(request)
|
||||
return CreateNamespaceResponse(
|
||||
properties=response.properties if hasattr(response, "properties") else None
|
||||
)
|
||||
@@ -1036,7 +1012,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
mode=_normalize_drop_namespace_mode(mode),
|
||||
behavior=_normalize_drop_namespace_behavior(behavior),
|
||||
)
|
||||
response = self._namespace_client.drop_namespace(request)
|
||||
response = self._ns.drop_namespace(request)
|
||||
return DropNamespaceResponse(
|
||||
properties=(
|
||||
response.properties if hasattr(response, "properties") else None
|
||||
@@ -1063,7 +1039,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
Response containing the namespace properties.
|
||||
"""
|
||||
request = DescribeNamespaceRequest(id=namespace)
|
||||
response = self._namespace_client.describe_namespace(request)
|
||||
response = self._ns.describe_namespace(request)
|
||||
return DescribeNamespaceResponse(
|
||||
properties=response.properties if hasattr(response, "properties") else None
|
||||
)
|
||||
@@ -1096,7 +1072,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
if namespace is None:
|
||||
namespace = []
|
||||
request = ListTablesRequest(id=namespace, page_token=page_token, limit=limit)
|
||||
response = self._namespace_client.list_tables(request)
|
||||
response = self._ns.list_tables(request)
|
||||
return ListTablesResponse(
|
||||
tables=response.tables if response.tables else [],
|
||||
page_token=response.page_token,
|
||||
|
||||
@@ -1746,8 +1746,6 @@ class LanceTable(Table):
|
||||
storage_options_provider: Optional["StorageOptionsProvider"] = None,
|
||||
index_cache_size: Optional[int] = None,
|
||||
location: Optional[str] = None,
|
||||
namespace_client: Optional[Any] = None,
|
||||
managed_versioning: Optional[bool] = None,
|
||||
_async: AsyncTable = None,
|
||||
):
|
||||
if namespace is None:
|
||||
@@ -1755,7 +1753,6 @@ class LanceTable(Table):
|
||||
self._conn = connection
|
||||
self._namespace = namespace
|
||||
self._location = location # Store location for use in _dataset_path
|
||||
self._namespace_client = namespace_client
|
||||
if _async is not None:
|
||||
self._table = _async
|
||||
else:
|
||||
@@ -1767,8 +1764,6 @@ class LanceTable(Table):
|
||||
storage_options_provider=storage_options_provider,
|
||||
index_cache_size=index_cache_size,
|
||||
location=location,
|
||||
namespace_client=namespace_client,
|
||||
managed_versioning=managed_versioning,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -1811,8 +1806,6 @@ class LanceTable(Table):
|
||||
storage_options_provider: Optional["StorageOptionsProvider"] = None,
|
||||
index_cache_size: Optional[int] = None,
|
||||
location: Optional[str] = None,
|
||||
namespace_client: Optional[Any] = None,
|
||||
managed_versioning: Optional[bool] = None,
|
||||
):
|
||||
if namespace is None:
|
||||
namespace = []
|
||||
@@ -1824,8 +1817,6 @@ class LanceTable(Table):
|
||||
storage_options_provider=storage_options_provider,
|
||||
index_cache_size=index_cache_size,
|
||||
location=location,
|
||||
namespace_client=namespace_client,
|
||||
managed_versioning=managed_versioning,
|
||||
)
|
||||
|
||||
# check the dataset exists
|
||||
@@ -1857,16 +1848,6 @@ class LanceTable(Table):
|
||||
"Please install with `pip install pylance`."
|
||||
)
|
||||
|
||||
if self._namespace_client is not None:
|
||||
table_id = self._namespace + [self.name]
|
||||
return lance.dataset(
|
||||
version=self.version,
|
||||
storage_options=self._conn.storage_options,
|
||||
namespace=self._namespace_client,
|
||||
table_id=table_id,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
return lance.dataset(
|
||||
self._dataset_path,
|
||||
version=self.version,
|
||||
@@ -2732,7 +2713,6 @@ class LanceTable(Table):
|
||||
data_storage_version: Optional[str] = None,
|
||||
enable_v2_manifest_paths: Optional[bool] = None,
|
||||
location: Optional[str] = None,
|
||||
namespace_client: Optional[Any] = None,
|
||||
):
|
||||
"""
|
||||
Create a new table.
|
||||
@@ -2793,7 +2773,6 @@ class LanceTable(Table):
|
||||
self._conn = db
|
||||
self._namespace = namespace
|
||||
self._location = location
|
||||
self._namespace_client = namespace_client
|
||||
|
||||
if data_storage_version is not None:
|
||||
warnings.warn(
|
||||
|
||||
@@ -326,24 +326,6 @@ def test_add_struct(mem_db: DBConnection):
|
||||
table = mem_db.create_table("test2", schema=schema)
|
||||
table.add(data)
|
||||
|
||||
struct_type = pa.struct(
|
||||
[
|
||||
("b", pa.int64()),
|
||||
("a", pa.int64()),
|
||||
]
|
||||
)
|
||||
expected = pa.table(
|
||||
{
|
||||
"s_list": [
|
||||
[
|
||||
pa.scalar({"b": 1, "a": 2}, type=struct_type),
|
||||
pa.scalar({"b": 4, "a": None}, type=struct_type),
|
||||
]
|
||||
],
|
||||
}
|
||||
)
|
||||
assert table.to_arrow() == expected
|
||||
|
||||
|
||||
def test_add_subschema(mem_db: DBConnection):
|
||||
schema = pa.schema(
|
||||
|
||||
@@ -17,8 +17,7 @@ use pyo3::{
|
||||
use pyo3_async_runtimes::tokio::future_into_py;
|
||||
|
||||
use crate::{
|
||||
error::PythonErrorExt, namespace::extract_namespace_arc,
|
||||
storage_options::py_object_to_storage_options_provider, table::Table,
|
||||
error::PythonErrorExt, storage_options::py_object_to_storage_options_provider, table::Table,
|
||||
};
|
||||
|
||||
#[pyclass]
|
||||
@@ -183,8 +182,7 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (name, namespace=vec![], storage_options = None, storage_options_provider=None, index_cache_size = None, location=None, namespace_client=None, managed_versioning=None))]
|
||||
#[pyo3(signature = (name, namespace=vec![], storage_options = None, storage_options_provider=None, index_cache_size = None, location=None))]
|
||||
pub fn open_table(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
@@ -193,13 +191,11 @@ impl Connection {
|
||||
storage_options_provider: Option<Py<PyAny>>,
|
||||
index_cache_size: Option<u32>,
|
||||
location: Option<String>,
|
||||
namespace_client: Option<Py<PyAny>>,
|
||||
managed_versioning: Option<bool>,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
|
||||
let mut builder = inner.open_table(name);
|
||||
builder = builder.namespace(namespace.clone());
|
||||
builder = builder.namespace(namespace);
|
||||
if let Some(storage_options) = storage_options {
|
||||
builder = builder.storage_options(storage_options);
|
||||
}
|
||||
@@ -213,20 +209,6 @@ impl Connection {
|
||||
if let Some(location) = location {
|
||||
builder = builder.location(location);
|
||||
}
|
||||
// Extract namespace client from Python object if provided
|
||||
let ns_client = if let Some(ns_obj) = namespace_client {
|
||||
let py = self_.py();
|
||||
Some(extract_namespace_arc(py, ns_obj)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(ns_client) = ns_client {
|
||||
builder = builder.namespace_client(ns_client);
|
||||
}
|
||||
// Pass managed_versioning if provided to avoid redundant describe_table call
|
||||
if let Some(enabled) = managed_versioning {
|
||||
builder = builder.managed_versioning(enabled);
|
||||
}
|
||||
|
||||
future_into_py(self_.py(), async move {
|
||||
let table = builder.execute().await.infer_error()?;
|
||||
|
||||
@@ -23,7 +23,6 @@ pub mod connection;
|
||||
pub mod error;
|
||||
pub mod header;
|
||||
pub mod index;
|
||||
pub mod namespace;
|
||||
pub mod permutation;
|
||||
pub mod query;
|
||||
pub mod session;
|
||||
|
||||
@@ -1,746 +0,0 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
//! Namespace utilities for Python bindings
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use bytes::Bytes;
|
||||
use lance_namespace::LanceNamespace as LanceNamespaceTrait;
|
||||
use lance_namespace::models::*;
|
||||
use pyo3::prelude::*;
|
||||
use pyo3::types::PyDict;
|
||||
|
||||
/// Wrapper that allows any Python object implementing LanceNamespace protocol
|
||||
/// to be used as a Rust LanceNamespace.
|
||||
///
|
||||
/// This is similar to PyLanceNamespace in lance's Python bindings - it wraps a Python
|
||||
/// object and calls back into Python when namespace methods are invoked.
|
||||
pub struct PyLanceNamespace {
|
||||
py_namespace: Arc<Py<PyAny>>,
|
||||
namespace_id: String,
|
||||
}
|
||||
|
||||
impl PyLanceNamespace {
|
||||
/// Create a new PyLanceNamespace wrapper around a Python namespace object.
|
||||
pub fn new(_py: Python<'_>, py_namespace: &Bound<'_, PyAny>) -> PyResult<Self> {
|
||||
let namespace_id = py_namespace
|
||||
.call_method0("namespace_id")?
|
||||
.extract::<String>()?;
|
||||
|
||||
Ok(Self {
|
||||
py_namespace: Arc::new(py_namespace.clone().unbind()),
|
||||
namespace_id,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create an Arc<dyn LanceNamespace> from a Python namespace object.
|
||||
pub fn create_arc(
|
||||
py: Python<'_>,
|
||||
py_namespace: &Bound<'_, PyAny>,
|
||||
) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
|
||||
let wrapper = Self::new(py, py_namespace)?;
|
||||
Ok(Arc::new(wrapper))
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PyLanceNamespace {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "PyLanceNamespace {{ id: {} }}", self.namespace_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Get or create the DictWithModelDump class in Python.
|
||||
/// This class acts like a dict but also has model_dump() method.
|
||||
/// This allows it to work with both:
|
||||
/// - depythonize (which expects a dict/Mapping)
|
||||
/// - Python code that calls .model_dump() (like DirectoryNamespace wrapper)
|
||||
fn get_dict_with_model_dump_class(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
// Use a module-level cache via __builtins__
|
||||
let builtins = py.import("builtins")?;
|
||||
if builtins.hasattr("_DictWithModelDump")? {
|
||||
return builtins.getattr("_DictWithModelDump");
|
||||
}
|
||||
|
||||
// Create the class using exec
|
||||
let locals = PyDict::new(py);
|
||||
py.run(
|
||||
c"class DictWithModelDump(dict):
|
||||
def model_dump(self):
|
||||
return dict(self)",
|
||||
None,
|
||||
Some(&locals),
|
||||
)?;
|
||||
let class = locals.get_item("DictWithModelDump")?.ok_or_else(|| {
|
||||
pyo3::exceptions::PyRuntimeError::new_err("Failed to create DictWithModelDump class")
|
||||
})?;
|
||||
|
||||
// Cache it
|
||||
builtins.setattr("_DictWithModelDump", &class)?;
|
||||
Ok(class)
|
||||
}
|
||||
|
||||
/// Helper to call a Python namespace method with JSON serialization.
|
||||
/// For methods that take a request and return a response.
|
||||
/// Uses DictWithModelDump to pass a dict that also has model_dump() method,
|
||||
/// making it compatible with both depythonize and Python wrappers.
|
||||
async fn call_py_method<Req, Resp>(
|
||||
py_namespace: Arc<Py<PyAny>>,
|
||||
method_name: &'static str,
|
||||
request: Req,
|
||||
) -> lance_core::Result<Resp>
|
||||
where
|
||||
Req: serde::Serialize + Send + 'static,
|
||||
Resp: serde::de::DeserializeOwned + Send + 'static,
|
||||
{
|
||||
let request_json = serde_json::to_string(&request).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to serialize request for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
let response_json = tokio::task::spawn_blocking(move || {
|
||||
Python::attach(|py| {
|
||||
let json_module = py.import("json")?;
|
||||
let request_dict = json_module.call_method1("loads", (&request_json,))?;
|
||||
|
||||
// Wrap dict in DictWithModelDump so it works with both depythonize and .model_dump()
|
||||
let dict_class = get_dict_with_model_dump_class(py)?;
|
||||
let request_arg = dict_class.call1((request_dict,))?;
|
||||
|
||||
// Call the Python method
|
||||
let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
|
||||
|
||||
// Convert response to dict, then to JSON
|
||||
// Pydantic models have model_dump() method
|
||||
let result_dict = if result.bind(py).hasattr("model_dump")? {
|
||||
result.call_method0(py, "model_dump")?
|
||||
} else {
|
||||
result
|
||||
};
|
||||
let response_json: String = json_module
|
||||
.call_method1("dumps", (result_dict,))?
|
||||
.extract()?;
|
||||
Ok::<_, PyErr>(response_json)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Task join error for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?
|
||||
.map_err(|e: PyErr| {
|
||||
lance_core::Error::io(
|
||||
format!("Python error in {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
serde_json::from_str(&response_json).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to deserialize response from {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Helper for methods that return () on success
|
||||
async fn call_py_method_unit<Req>(
|
||||
py_namespace: Arc<Py<PyAny>>,
|
||||
method_name: &'static str,
|
||||
request: Req,
|
||||
) -> lance_core::Result<()>
|
||||
where
|
||||
Req: serde::Serialize + Send + 'static,
|
||||
{
|
||||
let request_json = serde_json::to_string(&request).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to serialize request for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
Python::attach(|py| {
|
||||
let json_module = py.import("json")?;
|
||||
let request_dict = json_module.call_method1("loads", (&request_json,))?;
|
||||
|
||||
// Wrap dict in DictWithModelDump
|
||||
let dict_class = get_dict_with_model_dump_class(py)?;
|
||||
let request_arg = dict_class.call1((request_dict,))?;
|
||||
|
||||
// Call the Python method
|
||||
py_namespace.call_method1(py, method_name, (request_arg,))?;
|
||||
Ok::<_, PyErr>(())
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Task join error for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?
|
||||
.map_err(|e: PyErr| {
|
||||
lance_core::Error::io(
|
||||
format!("Python error in {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Helper for methods that return a primitive type
|
||||
async fn call_py_method_primitive<Req, Resp>(
|
||||
py_namespace: Arc<Py<PyAny>>,
|
||||
method_name: &'static str,
|
||||
request: Req,
|
||||
) -> lance_core::Result<Resp>
|
||||
where
|
||||
Req: serde::Serialize + Send + 'static,
|
||||
Resp: for<'py> pyo3::FromPyObject<'py> + Send + 'static,
|
||||
{
|
||||
let request_json = serde_json::to_string(&request).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to serialize request for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
Python::attach(|py| {
|
||||
let json_module = py.import("json")?;
|
||||
let request_dict = json_module.call_method1("loads", (&request_json,))?;
|
||||
|
||||
// Wrap dict in DictWithModelDump
|
||||
let dict_class = get_dict_with_model_dump_class(py)?;
|
||||
let request_arg = dict_class.call1((request_dict,))?;
|
||||
|
||||
// Call the Python method
|
||||
let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
|
||||
let value: Resp = result.extract(py)?;
|
||||
Ok::<_, PyErr>(value)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Task join error for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?
|
||||
.map_err(|e: PyErr| {
|
||||
lance_core::Error::io(
|
||||
format!("Python error in {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Helper for methods that return Bytes
|
||||
async fn call_py_method_bytes<Req>(
|
||||
py_namespace: Arc<Py<PyAny>>,
|
||||
method_name: &'static str,
|
||||
request: Req,
|
||||
) -> lance_core::Result<Bytes>
|
||||
where
|
||||
Req: serde::Serialize + Send + 'static,
|
||||
{
|
||||
let request_json = serde_json::to_string(&request).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to serialize request for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
Python::attach(|py| {
|
||||
let json_module = py.import("json")?;
|
||||
let request_dict = json_module.call_method1("loads", (&request_json,))?;
|
||||
|
||||
// Wrap dict in DictWithModelDump
|
||||
let dict_class = get_dict_with_model_dump_class(py)?;
|
||||
let request_arg = dict_class.call1((request_dict,))?;
|
||||
|
||||
// Call the Python method
|
||||
let result = py_namespace.call_method1(py, method_name, (request_arg,))?;
|
||||
let bytes_data: Vec<u8> = result.extract(py)?;
|
||||
Ok::<_, PyErr>(Bytes::from(bytes_data))
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Task join error for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?
|
||||
.map_err(|e: PyErr| {
|
||||
lance_core::Error::io(
|
||||
format!("Python error in {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/// Helper for methods that take request + data and return a response
|
||||
async fn call_py_method_with_data<Req, Resp>(
|
||||
py_namespace: Arc<Py<PyAny>>,
|
||||
method_name: &'static str,
|
||||
request: Req,
|
||||
data: Bytes,
|
||||
) -> lance_core::Result<Resp>
|
||||
where
|
||||
Req: serde::Serialize + Send + 'static,
|
||||
Resp: serde::de::DeserializeOwned + Send + 'static,
|
||||
{
|
||||
let request_json = serde_json::to_string(&request).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to serialize request for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
let response_json = tokio::task::spawn_blocking(move || {
|
||||
Python::attach(|py| {
|
||||
let json_module = py.import("json")?;
|
||||
let request_dict = json_module.call_method1("loads", (&request_json,))?;
|
||||
|
||||
// Wrap dict in DictWithModelDump
|
||||
let dict_class = get_dict_with_model_dump_class(py)?;
|
||||
let request_arg = dict_class.call1((request_dict,))?;
|
||||
|
||||
// Pass request and bytes to Python method
|
||||
let py_bytes = pyo3::types::PyBytes::new(py, &data);
|
||||
let result = py_namespace.call_method1(py, method_name, (request_arg, py_bytes))?;
|
||||
|
||||
// Convert response dict to JSON
|
||||
let response_json: String = json_module.call_method1("dumps", (result,))?.extract()?;
|
||||
Ok::<_, PyErr>(response_json)
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Task join error for {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?
|
||||
.map_err(|e: PyErr| {
|
||||
lance_core::Error::io(
|
||||
format!("Python error in {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})?;
|
||||
|
||||
serde_json::from_str(&response_json).map_err(|e| {
|
||||
lance_core::Error::io(
|
||||
format!("Failed to deserialize response from {}: {}", method_name, e),
|
||||
Default::default(),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LanceNamespaceTrait for PyLanceNamespace {
|
||||
fn namespace_id(&self) -> String {
|
||||
self.namespace_id.clone()
|
||||
}
|
||||
|
||||
async fn list_namespaces(
|
||||
&self,
|
||||
request: ListNamespacesRequest,
|
||||
) -> lance_core::Result<ListNamespacesResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "list_namespaces", request).await
|
||||
}
|
||||
|
||||
async fn describe_namespace(
|
||||
&self,
|
||||
request: DescribeNamespaceRequest,
|
||||
) -> lance_core::Result<DescribeNamespaceResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "describe_namespace", request).await
|
||||
}
|
||||
|
||||
async fn create_namespace(
|
||||
&self,
|
||||
request: CreateNamespaceRequest,
|
||||
) -> lance_core::Result<CreateNamespaceResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "create_namespace", request).await
|
||||
}
|
||||
|
||||
async fn drop_namespace(
|
||||
&self,
|
||||
request: DropNamespaceRequest,
|
||||
) -> lance_core::Result<DropNamespaceResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "drop_namespace", request).await
|
||||
}
|
||||
|
||||
async fn namespace_exists(&self, request: NamespaceExistsRequest) -> lance_core::Result<()> {
|
||||
call_py_method_unit(self.py_namespace.clone(), "namespace_exists", request).await
|
||||
}
|
||||
|
||||
async fn list_tables(
|
||||
&self,
|
||||
request: ListTablesRequest,
|
||||
) -> lance_core::Result<ListTablesResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "list_tables", request).await
|
||||
}
|
||||
|
||||
async fn describe_table(
|
||||
&self,
|
||||
request: DescribeTableRequest,
|
||||
) -> lance_core::Result<DescribeTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "describe_table", request).await
|
||||
}
|
||||
|
||||
async fn register_table(
|
||||
&self,
|
||||
request: RegisterTableRequest,
|
||||
) -> lance_core::Result<RegisterTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "register_table", request).await
|
||||
}
|
||||
|
||||
async fn table_exists(&self, request: TableExistsRequest) -> lance_core::Result<()> {
|
||||
call_py_method_unit(self.py_namespace.clone(), "table_exists", request).await
|
||||
}
|
||||
|
||||
async fn drop_table(&self, request: DropTableRequest) -> lance_core::Result<DropTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "drop_table", request).await
|
||||
}
|
||||
|
||||
async fn deregister_table(
|
||||
&self,
|
||||
request: DeregisterTableRequest,
|
||||
) -> lance_core::Result<DeregisterTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "deregister_table", request).await
|
||||
}
|
||||
|
||||
async fn count_table_rows(&self, request: CountTableRowsRequest) -> lance_core::Result<i64> {
|
||||
call_py_method_primitive(self.py_namespace.clone(), "count_table_rows", request).await
|
||||
}
|
||||
|
||||
async fn create_table(
|
||||
&self,
|
||||
request: CreateTableRequest,
|
||||
request_data: Bytes,
|
||||
) -> lance_core::Result<CreateTableResponse> {
|
||||
call_py_method_with_data(
|
||||
self.py_namespace.clone(),
|
||||
"create_table",
|
||||
request,
|
||||
request_data,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn declare_table(
|
||||
&self,
|
||||
request: DeclareTableRequest,
|
||||
) -> lance_core::Result<DeclareTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "declare_table", request).await
|
||||
}
|
||||
|
||||
async fn insert_into_table(
|
||||
&self,
|
||||
request: InsertIntoTableRequest,
|
||||
request_data: Bytes,
|
||||
) -> lance_core::Result<InsertIntoTableResponse> {
|
||||
call_py_method_with_data(
|
||||
self.py_namespace.clone(),
|
||||
"insert_into_table",
|
||||
request,
|
||||
request_data,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn merge_insert_into_table(
|
||||
&self,
|
||||
request: MergeInsertIntoTableRequest,
|
||||
request_data: Bytes,
|
||||
) -> lance_core::Result<MergeInsertIntoTableResponse> {
|
||||
call_py_method_with_data(
|
||||
self.py_namespace.clone(),
|
||||
"merge_insert_into_table",
|
||||
request,
|
||||
request_data,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_table(
|
||||
&self,
|
||||
request: UpdateTableRequest,
|
||||
) -> lance_core::Result<UpdateTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "update_table", request).await
|
||||
}
|
||||
|
||||
async fn delete_from_table(
|
||||
&self,
|
||||
request: DeleteFromTableRequest,
|
||||
) -> lance_core::Result<DeleteFromTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "delete_from_table", request).await
|
||||
}
|
||||
|
||||
async fn query_table(&self, request: QueryTableRequest) -> lance_core::Result<Bytes> {
|
||||
call_py_method_bytes(self.py_namespace.clone(), "query_table", request).await
|
||||
}
|
||||
|
||||
async fn create_table_index(
|
||||
&self,
|
||||
request: CreateTableIndexRequest,
|
||||
) -> lance_core::Result<CreateTableIndexResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "create_table_index", request).await
|
||||
}
|
||||
|
||||
async fn list_table_indices(
|
||||
&self,
|
||||
request: ListTableIndicesRequest,
|
||||
) -> lance_core::Result<ListTableIndicesResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "list_table_indices", request).await
|
||||
}
|
||||
|
||||
async fn describe_table_index_stats(
|
||||
&self,
|
||||
request: DescribeTableIndexStatsRequest,
|
||||
) -> lance_core::Result<DescribeTableIndexStatsResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"describe_table_index_stats",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn describe_transaction(
|
||||
&self,
|
||||
request: DescribeTransactionRequest,
|
||||
) -> lance_core::Result<DescribeTransactionResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "describe_transaction", request).await
|
||||
}
|
||||
|
||||
async fn alter_transaction(
|
||||
&self,
|
||||
request: AlterTransactionRequest,
|
||||
) -> lance_core::Result<AlterTransactionResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "alter_transaction", request).await
|
||||
}
|
||||
|
||||
async fn create_table_scalar_index(
|
||||
&self,
|
||||
request: CreateTableIndexRequest,
|
||||
) -> lance_core::Result<CreateTableScalarIndexResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"create_table_scalar_index",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn drop_table_index(
|
||||
&self,
|
||||
request: DropTableIndexRequest,
|
||||
) -> lance_core::Result<DropTableIndexResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "drop_table_index", request).await
|
||||
}
|
||||
|
||||
async fn list_all_tables(
|
||||
&self,
|
||||
request: ListTablesRequest,
|
||||
) -> lance_core::Result<ListTablesResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "list_all_tables", request).await
|
||||
}
|
||||
|
||||
async fn restore_table(
|
||||
&self,
|
||||
request: RestoreTableRequest,
|
||||
) -> lance_core::Result<RestoreTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "restore_table", request).await
|
||||
}
|
||||
|
||||
async fn rename_table(
|
||||
&self,
|
||||
request: RenameTableRequest,
|
||||
) -> lance_core::Result<RenameTableResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "rename_table", request).await
|
||||
}
|
||||
|
||||
async fn list_table_versions(
|
||||
&self,
|
||||
request: ListTableVersionsRequest,
|
||||
) -> lance_core::Result<ListTableVersionsResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "list_table_versions", request).await
|
||||
}
|
||||
|
||||
async fn create_table_version(
|
||||
&self,
|
||||
request: CreateTableVersionRequest,
|
||||
) -> lance_core::Result<CreateTableVersionResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "create_table_version", request).await
|
||||
}
|
||||
|
||||
async fn describe_table_version(
|
||||
&self,
|
||||
request: DescribeTableVersionRequest,
|
||||
) -> lance_core::Result<DescribeTableVersionResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "describe_table_version", request).await
|
||||
}
|
||||
|
||||
async fn batch_delete_table_versions(
|
||||
&self,
|
||||
request: BatchDeleteTableVersionsRequest,
|
||||
) -> lance_core::Result<BatchDeleteTableVersionsResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"batch_delete_table_versions",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_table_schema_metadata(
|
||||
&self,
|
||||
request: UpdateTableSchemaMetadataRequest,
|
||||
) -> lance_core::Result<UpdateTableSchemaMetadataResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"update_table_schema_metadata",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_table_stats(
|
||||
&self,
|
||||
request: GetTableStatsRequest,
|
||||
) -> lance_core::Result<GetTableStatsResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "get_table_stats", request).await
|
||||
}
|
||||
|
||||
async fn explain_table_query_plan(
|
||||
&self,
|
||||
request: ExplainTableQueryPlanRequest,
|
||||
) -> lance_core::Result<String> {
|
||||
call_py_method_primitive(
|
||||
self.py_namespace.clone(),
|
||||
"explain_table_query_plan",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn analyze_table_query_plan(
|
||||
&self,
|
||||
request: AnalyzeTableQueryPlanRequest,
|
||||
) -> lance_core::Result<String> {
|
||||
call_py_method_primitive(
|
||||
self.py_namespace.clone(),
|
||||
"analyze_table_query_plan",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn alter_table_add_columns(
|
||||
&self,
|
||||
request: AlterTableAddColumnsRequest,
|
||||
) -> lance_core::Result<AlterTableAddColumnsResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"alter_table_add_columns",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn alter_table_alter_columns(
|
||||
&self,
|
||||
request: AlterTableAlterColumnsRequest,
|
||||
) -> lance_core::Result<AlterTableAlterColumnsResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"alter_table_alter_columns",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn alter_table_drop_columns(
|
||||
&self,
|
||||
request: AlterTableDropColumnsRequest,
|
||||
) -> lance_core::Result<AlterTableDropColumnsResponse> {
|
||||
call_py_method(
|
||||
self.py_namespace.clone(),
|
||||
"alter_table_drop_columns",
|
||||
request,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_table_tags(
|
||||
&self,
|
||||
request: ListTableTagsRequest,
|
||||
) -> lance_core::Result<ListTableTagsResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "list_table_tags", request).await
|
||||
}
|
||||
|
||||
async fn create_table_tag(
|
||||
&self,
|
||||
request: CreateTableTagRequest,
|
||||
) -> lance_core::Result<CreateTableTagResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "create_table_tag", request).await
|
||||
}
|
||||
|
||||
async fn delete_table_tag(
|
||||
&self,
|
||||
request: DeleteTableTagRequest,
|
||||
) -> lance_core::Result<DeleteTableTagResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "delete_table_tag", request).await
|
||||
}
|
||||
|
||||
async fn update_table_tag(
|
||||
&self,
|
||||
request: UpdateTableTagRequest,
|
||||
) -> lance_core::Result<UpdateTableTagResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "update_table_tag", request).await
|
||||
}
|
||||
|
||||
async fn get_table_tag_version(
|
||||
&self,
|
||||
request: GetTableTagVersionRequest,
|
||||
) -> lance_core::Result<GetTableTagVersionResponse> {
|
||||
call_py_method(self.py_namespace.clone(), "get_table_tag_version", request).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert Python dict to HashMap<String, String>
|
||||
#[allow(dead_code)]
|
||||
fn dict_to_hashmap(dict: &Bound<'_, PyDict>) -> PyResult<HashMap<String, String>> {
|
||||
let mut map = HashMap::new();
|
||||
for (key, value) in dict.iter() {
|
||||
let key_str: String = key.extract()?;
|
||||
let value_str: String = value.extract()?;
|
||||
map.insert(key_str, value_str);
|
||||
}
|
||||
Ok(map)
|
||||
}
|
||||
|
||||
/// Extract an Arc<dyn LanceNamespace> from a Python namespace object.
|
||||
///
|
||||
/// This function wraps any Python namespace object with PyLanceNamespace.
|
||||
/// The PyLanceNamespace wrapper uses DictWithModelDump to pass requests,
|
||||
/// which works with both:
|
||||
/// - Native namespaces (DirectoryNamespace, RestNamespace) that use depythonize (expects dict)
|
||||
/// - Custom Python implementations that call .model_dump() on the request
|
||||
pub fn extract_namespace_arc(
|
||||
py: Python<'_>,
|
||||
ns: Py<PyAny>,
|
||||
) -> PyResult<Arc<dyn LanceNamespaceTrait>> {
|
||||
let ns_ref = ns.bind(py);
|
||||
PyLanceNamespace::create_arc(py, ns_ref)
|
||||
}
|
||||
@@ -66,12 +66,11 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
|
||||
.inner
|
||||
.bind(py)
|
||||
.call_method0("fetch_storage_options")
|
||||
.map_err(|e| lance_core::Error::IO {
|
||||
source: Box::new(std::io::Error::other(format!(
|
||||
.map_err(|e| {
|
||||
lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
|
||||
"Failed to call fetch_storage_options: {}",
|
||||
e
|
||||
))),
|
||||
location: snafu::location!(),
|
||||
))))
|
||||
})?;
|
||||
|
||||
// If result is None, return None
|
||||
@@ -81,26 +80,25 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
|
||||
|
||||
// Extract the result dict - should be a flat Map<String, String>
|
||||
let result_dict = result.downcast::<PyDict>().map_err(|_| {
|
||||
lance_core::Error::InvalidInput {
|
||||
source: "fetch_storage_options() must return None or a dict of string key-value pairs".into(),
|
||||
location: snafu::location!(),
|
||||
}
|
||||
lance_core::Error::invalid_input(
|
||||
"fetch_storage_options() must return None or a dict of string key-value pairs",
|
||||
)
|
||||
})?;
|
||||
|
||||
// Convert all entries to HashMap<String, String>
|
||||
let mut storage_options = HashMap::new();
|
||||
for (key, value) in result_dict.iter() {
|
||||
let key_str: String = key.extract().map_err(|e| {
|
||||
lance_core::Error::InvalidInput {
|
||||
source: format!("Storage option key must be a string: {}", e).into(),
|
||||
location: snafu::location!(),
|
||||
}
|
||||
lance_core::Error::invalid_input(format!(
|
||||
"Storage option key must be a string: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
let value_str: String = value.extract().map_err(|e| {
|
||||
lance_core::Error::InvalidInput {
|
||||
source: format!("Storage option value must be a string: {}", e).into(),
|
||||
location: snafu::location!(),
|
||||
}
|
||||
lance_core::Error::invalid_input(format!(
|
||||
"Storage option value must be a string: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
storage_options.insert(key_str, value_str);
|
||||
}
|
||||
@@ -109,13 +107,10 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| lance_core::Error::IO {
|
||||
source: Box::new(std::io::Error::other(format!(
|
||||
.map_err(|e| lance_core::Error::io_source(Box::new(std::io::Error::other(format!(
|
||||
"Task join error: {}",
|
||||
e
|
||||
))),
|
||||
location: snafu::location!(),
|
||||
})?
|
||||
)))))?
|
||||
}
|
||||
|
||||
fn provider_id(&self) -> String {
|
||||
|
||||
4
python/uv.lock
generated
4
python/uv.lock
generated
@@ -2006,7 +2006,7 @@ requires-dist = [
|
||||
{ name = "botocore", marker = "extra == 'embeddings'", specifier = ">=1.31.57" },
|
||||
{ name = "cohere", marker = "extra == 'embeddings'" },
|
||||
{ name = "colpali-engine", marker = "extra == 'embeddings'", specifier = ">=0.3.10" },
|
||||
{ name = "datafusion", marker = "extra == 'tests'", specifier = "<52" },
|
||||
{ name = "datafusion", marker = "extra == 'tests'" },
|
||||
{ name = "deprecation" },
|
||||
{ name = "duckdb", marker = "extra == 'tests'" },
|
||||
{ name = "google-generativeai", marker = "extra == 'embeddings'" },
|
||||
@@ -2035,7 +2035,7 @@ requires-dist = [
|
||||
{ name = "pyarrow-stubs", marker = "extra == 'tests'" },
|
||||
{ name = "pydantic", specifier = ">=1.10" },
|
||||
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=1.0.0b14" },
|
||||
{ name = "pylance", marker = "extra == 'tests'", specifier = ">=1.0.0b14,<3.0.0" },
|
||||
{ name = "pylance", marker = "extra == 'tests'", specifier = ">=1.0.0b14" },
|
||||
{ name = "pyright", marker = "extra == 'dev'" },
|
||||
{ name = "pytest", marker = "extra == 'tests'" },
|
||||
{ name = "pytest-asyncio", marker = "extra == 'tests'" },
|
||||
|
||||
@@ -136,7 +136,6 @@ impl OpenTableBuilder {
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
},
|
||||
embedding_registry,
|
||||
}
|
||||
@@ -236,29 +235,6 @@ impl OpenTableBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set a namespace client for managed versioning support.
|
||||
///
|
||||
/// When a namespace client is provided and the table has `managed_versioning` enabled,
|
||||
/// the table will use the namespace's commit handler to notify the namespace of
|
||||
/// version changes. This enables features like event emission for table modifications.
|
||||
pub fn namespace_client(mut self, client: Arc<dyn lance_namespace::LanceNamespace>) -> Self {
|
||||
self.request.namespace_client = Some(client);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether managed versioning is enabled for this table.
|
||||
///
|
||||
/// When set to `Some(true)`, the table will use namespace-managed commits.
|
||||
/// When set to `Some(false)`, the table will use local commits even if namespace_client is set.
|
||||
/// When set to `None` (default), the value will be fetched from the namespace if namespace_client is set.
|
||||
///
|
||||
/// This is typically set when the caller has already queried the namespace and knows the
|
||||
/// managed_versioning status, avoiding a redundant describe_table call.
|
||||
pub fn managed_versioning(mut self, enabled: bool) -> Self {
|
||||
self.request.managed_versioning = Some(enabled);
|
||||
self
|
||||
}
|
||||
|
||||
/// Open the table
|
||||
pub async fn execute(self) -> Result<Table> {
|
||||
let table = self.parent.open_table(self.request).await?;
|
||||
@@ -318,12 +294,6 @@ impl CloneTableBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set a namespace client for managed versioning support.
|
||||
pub fn namespace_client(mut self, client: Arc<dyn lance_namespace::LanceNamespace>) -> Self {
|
||||
self.request.namespace_client = Some(client);
|
||||
self
|
||||
}
|
||||
|
||||
/// Execute the clone operation
|
||||
pub async fn execute(self) -> Result<Table> {
|
||||
let parent = self.parent.clone();
|
||||
@@ -596,11 +566,8 @@ pub struct ConnectBuilder {
|
||||
}
|
||||
|
||||
#[cfg(feature = "remote")]
|
||||
const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 3] = [
|
||||
("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name"),
|
||||
("AZURE_CLIENT_ID", "azure_client_id"),
|
||||
("AZURE_TENANT_ID", "azure_tenant_id"),
|
||||
];
|
||||
const ENV_VARS_TO_STORAGE_OPTS: [(&str, &str); 1] =
|
||||
[("AZURE_STORAGE_ACCOUNT_NAME", "azure_storage_account_name")];
|
||||
|
||||
impl ConnectBuilder {
|
||||
/// Create a new [`ConnectOptions`] with the given database URI.
|
||||
|
||||
@@ -66,10 +66,6 @@ pub struct OpenTableRequest {
|
||||
/// Optional namespace client for server-side query execution.
|
||||
/// When set, queries will be executed on the namespace server instead of locally.
|
||||
pub namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
/// Whether managed versioning is enabled for this table.
|
||||
/// When Some(true), the table will use namespace-managed commits instead of local commits.
|
||||
/// When None and namespace_client is provided, the value will be fetched from the namespace.
|
||||
pub managed_versioning: Option<bool>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for OpenTableRequest {
|
||||
@@ -81,7 +77,6 @@ impl std::fmt::Debug for OpenTableRequest {
|
||||
.field("lance_read_params", &self.lance_read_params)
|
||||
.field("location", &self.location)
|
||||
.field("namespace_client", &self.namespace_client)
|
||||
.field("managed_versioning", &self.managed_versioning)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -166,9 +161,6 @@ pub struct CloneTableRequest {
|
||||
/// Whether to perform a shallow clone (true) or deep clone (false). Defaults to true.
|
||||
/// Currently only shallow clone is supported.
|
||||
pub is_shallow: bool,
|
||||
/// Optional namespace client for managed versioning support.
|
||||
/// When set, enables the commit handler to track table versions through the namespace.
|
||||
pub namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
}
|
||||
|
||||
impl CloneTableRequest {
|
||||
@@ -180,7 +172,6 @@ impl CloneTableRequest {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -669,7 +669,6 @@ impl ListingDatabase {
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
};
|
||||
let req = (callback)(req);
|
||||
let table = self.open_table(req).await?;
|
||||
@@ -870,7 +869,6 @@ impl Database for ListingDatabase {
|
||||
Some(write_params),
|
||||
self.read_consistency_interval,
|
||||
request.namespace_client,
|
||||
false, // server_side_query_enabled - listing database doesn't support server-side queries
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -948,9 +946,7 @@ impl Database for ListingDatabase {
|
||||
self.store_wrapper.clone(),
|
||||
None,
|
||||
self.read_consistency_interval,
|
||||
request.namespace_client,
|
||||
false, // server_side_query_enabled - listing database doesn't support server-side queries
|
||||
None, // managed_versioning - will be queried if namespace_client is provided
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -1026,8 +1022,6 @@ impl Database for ListingDatabase {
|
||||
Some(read_params),
|
||||
self.read_consistency_interval,
|
||||
request.namespace_client,
|
||||
false, // server_side_query_enabled - listing database doesn't support server-side queries
|
||||
request.managed_versioning, // Pass through managed_versioning from request
|
||||
)
|
||||
.await?,
|
||||
);
|
||||
@@ -1168,7 +1162,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1229,7 +1222,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1289,7 +1281,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -1326,7 +1317,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: false, // Request deep clone
|
||||
namespace_client: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -1367,7 +1357,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -1408,7 +1397,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -1428,7 +1416,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -1465,7 +1452,6 @@ mod tests {
|
||||
source_version: Some(1),
|
||||
source_tag: Some("v1.0".to_string()),
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await;
|
||||
|
||||
@@ -1539,7 +1525,6 @@ mod tests {
|
||||
source_version: Some(initial_version),
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1618,7 +1603,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: Some("v1.0".to_string()),
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1670,7 +1654,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -1763,7 +1746,6 @@ mod tests {
|
||||
source_version: None,
|
||||
source_tag: None,
|
||||
is_shallow: true,
|
||||
namespace_client: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -7,7 +7,6 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
|
||||
use lance_io::object_store::{ObjectStoreParams, StorageOptionsAccessor};
|
||||
use lance_namespace::{
|
||||
LanceNamespace,
|
||||
@@ -19,8 +18,6 @@ use lance_namespace::{
|
||||
},
|
||||
};
|
||||
use lance_namespace_impls::ConnectBuilder;
|
||||
use lance_table::io::commit::CommitHandler;
|
||||
use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
|
||||
|
||||
use crate::database::ReadConsistency;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -208,55 +205,40 @@ impl Database for LanceNamespaceDatabase {
|
||||
let mut table_id = request.namespace.clone();
|
||||
table_id.push(request.name.clone());
|
||||
|
||||
// Declare table metadata through lance-namespace before creating data files.
|
||||
let declare_request = DeclareTableRequest {
|
||||
id: Some(table_id.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = self
|
||||
let declare_response = self
|
||||
.namespace
|
||||
.declare_table(declare_request)
|
||||
.await
|
||||
.map_err(|e| Error::Runtime {
|
||||
message: format!("Failed to declare table: {}", e),
|
||||
})?;
|
||||
|
||||
let location = response.location.ok_or_else(|| Error::Runtime {
|
||||
let location = declare_response.location.ok_or_else(|| Error::Runtime {
|
||||
message: "Table location is missing from declare_table response".to_string(),
|
||||
})?;
|
||||
|
||||
// Use storage options from response, fall back to self.storage_options
|
||||
let initial_storage_options = response
|
||||
let initial_storage_options = declare_response
|
||||
.storage_options
|
||||
.or_else(|| Some(self.storage_options.clone()))
|
||||
.filter(|o| !o.is_empty());
|
||||
|
||||
let managed_versioning = response.managed_versioning;
|
||||
|
||||
// Build write params with storage options and commit handler
|
||||
let mut params = request.write_options.lance_write_params.unwrap_or_default();
|
||||
|
||||
// Set up storage options if provided
|
||||
if let Some(storage_opts) = initial_storage_options {
|
||||
let write_params = if let Some(storage_opts) = initial_storage_options {
|
||||
let mut params = request.write_options.lance_write_params.unwrap_or_default();
|
||||
let store_params = params
|
||||
.store_params
|
||||
.get_or_insert_with(ObjectStoreParams::default);
|
||||
store_params.storage_options_accessor = Some(Arc::new(
|
||||
StorageOptionsAccessor::with_static_options(storage_opts),
|
||||
));
|
||||
}
|
||||
|
||||
// Set up commit handler when managed_versioning is enabled
|
||||
if managed_versioning == Some(true) {
|
||||
let external_store =
|
||||
LanceNamespaceExternalManifestStore::new(self.namespace.clone(), table_id.clone());
|
||||
let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
|
||||
external_manifest_store: Arc::new(external_store),
|
||||
});
|
||||
params.commit_handler = Some(commit_handler);
|
||||
}
|
||||
|
||||
let write_params = Some(params);
|
||||
Some(params)
|
||||
} else {
|
||||
request.write_options.lance_write_params
|
||||
};
|
||||
|
||||
let native_table = NativeTable::create_from_namespace(
|
||||
self.namespace.clone(),
|
||||
|
||||
@@ -446,23 +446,13 @@ impl<S: HttpSend> RestfulLanceDbClient<S> {
|
||||
})?,
|
||||
);
|
||||
}
|
||||
// Map azure storage options to x-azure-* headers.
|
||||
// The option key uses underscores (e.g. "azure_client_id") while the
|
||||
// header uses hyphens (e.g. "x-azure-client-id").
|
||||
let azure_opts: [(&str, &str); 3] = [
|
||||
("azure_storage_account_name", "x-azure-storage-account-name"),
|
||||
("azure_client_id", "x-azure-client-id"),
|
||||
("azure_tenant_id", "x-azure-tenant-id"),
|
||||
];
|
||||
for (opt_key, header_name) in azure_opts {
|
||||
if let Some(v) = options.0.get(opt_key) {
|
||||
headers.insert(
|
||||
HeaderName::from_static(header_name),
|
||||
HeaderValue::from_str(v).map_err(|_| Error::InvalidInput {
|
||||
message: format!("non-ascii value '{}' for option '{}'", v, opt_key),
|
||||
})?,
|
||||
);
|
||||
}
|
||||
if let Some(v) = options.0.get("azure_storage_account_name") {
|
||||
headers.insert(
|
||||
HeaderName::from_static("x-azure-storage-account-name"),
|
||||
HeaderValue::from_str(v).map_err(|_| Error::InvalidInput {
|
||||
message: format!("non-ascii storage account name '{}' provided", db_name),
|
||||
})?,
|
||||
);
|
||||
}
|
||||
|
||||
for (key, value) in &config.extra_headers {
|
||||
@@ -1085,34 +1075,4 @@ mod tests {
|
||||
_ => panic!("Expected Runtime error"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_headers_azure_opts() {
|
||||
let mut opts = HashMap::new();
|
||||
opts.insert(
|
||||
"azure_storage_account_name".to_string(),
|
||||
"myaccount".to_string(),
|
||||
);
|
||||
opts.insert("azure_client_id".to_string(), "my-client-id".to_string());
|
||||
opts.insert("azure_tenant_id".to_string(), "my-tenant-id".to_string());
|
||||
let remote_opts = RemoteOptions::new(opts);
|
||||
|
||||
let headers = RestfulLanceDbClient::<Sender>::default_headers(
|
||||
"test-key",
|
||||
"us-east-1",
|
||||
"testdb",
|
||||
false,
|
||||
&remote_opts,
|
||||
None,
|
||||
&ClientConfig::default(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
headers.get("x-azure-storage-account-name").unwrap(),
|
||||
"myaccount"
|
||||
);
|
||||
assert_eq!(headers.get("x-azure-client-id").unwrap(), "my-client-id");
|
||||
assert_eq!(headers.get("x-azure-tenant-id").unwrap(), "my-tenant-id");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -464,7 +464,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
lance_read_params: None,
|
||||
location: None,
|
||||
namespace_client: None,
|
||||
managed_versioning: None,
|
||||
};
|
||||
let req = (callback)(req);
|
||||
self.open_table(req).await
|
||||
@@ -778,12 +777,7 @@ impl RemoteOptions {
|
||||
|
||||
impl From<StorageOptions> for RemoteOptions {
|
||||
fn from(options: StorageOptions) -> Self {
|
||||
let supported_opts = vec![
|
||||
"account_name",
|
||||
"azure_storage_account_name",
|
||||
"azure_client_id",
|
||||
"azure_tenant_id",
|
||||
];
|
||||
let supported_opts = vec!["account_name", "azure_storage_account_name"];
|
||||
let mut filtered = HashMap::new();
|
||||
for opt in supported_opts {
|
||||
if let Some(v) = options.0.get(opt) {
|
||||
|
||||
@@ -34,13 +34,9 @@ use lance_index::vector::sq::builder::SQBuildParams;
|
||||
use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsAccessor};
|
||||
pub use query::AnyQuery;
|
||||
|
||||
use lance::io::commit::namespace_manifest::LanceNamespaceExternalManifestStore;
|
||||
use lance_namespace::LanceNamespace;
|
||||
use lance_namespace::models::DescribeTableRequest;
|
||||
use lance_table::format::Manifest;
|
||||
use lance_table::io::commit::CommitHandler;
|
||||
use lance_table::io::commit::ManifestNamingScheme;
|
||||
use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::format;
|
||||
@@ -1216,13 +1212,10 @@ pub struct NativeTable {
|
||||
// This comes from the connection options. We store here so we can pass down
|
||||
// to the dataset when we recreate it (for example, in checkout_latest).
|
||||
read_consistency_interval: Option<std::time::Duration>,
|
||||
// Optional namespace client for namespace operations (e.g., managed versioning).
|
||||
// pub(crate) so query.rs can access the field for server-side query execution.
|
||||
// Optional namespace client for server-side query execution.
|
||||
// When set, queries will be executed on the namespace server instead of locally.
|
||||
// pub (crate) namespace_client so query.rs can access the fields
|
||||
pub(crate) namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
// Whether to enable server-side query execution via the namespace client.
|
||||
// When true and namespace_client is set, queries will be executed on the
|
||||
// namespace server instead of locally.
|
||||
pub(crate) server_side_query_enabled: bool,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for NativeTable {
|
||||
@@ -1234,7 +1227,6 @@ impl std::fmt::Debug for NativeTable {
|
||||
.field("uri", &self.uri)
|
||||
.field("read_consistency_interval", &self.read_consistency_interval)
|
||||
.field("namespace_client", &self.namespace_client)
|
||||
.field("server_side_query_enabled", &self.server_side_query_enabled)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -1271,7 +1263,7 @@ impl NativeTable {
|
||||
/// * A [NativeTable] object.
|
||||
pub async fn open(uri: &str) -> Result<Self> {
|
||||
let name = Self::get_table_name(uri)?;
|
||||
Self::open_with_params(uri, &name, vec![], None, None, None, None, false, None).await
|
||||
Self::open_with_params(uri, &name, vec![], None, None, None, None).await
|
||||
}
|
||||
|
||||
/// Opens an existing Table
|
||||
@@ -1281,10 +1273,7 @@ impl NativeTable {
|
||||
/// * `base_path` - The base path where the table is located
|
||||
/// * `name` The Table name
|
||||
/// * `params` The [ReadParams] to use when opening the table
|
||||
/// * `namespace_client` - Optional namespace client for namespace operations
|
||||
/// * `server_side_query_enabled` - Whether to enable server-side query execution
|
||||
/// * `managed_versioning` - Whether managed versioning is enabled. If None and namespace_client
|
||||
/// is provided, the value will be fetched via describe_table.
|
||||
/// * `namespace_client` - Optional namespace client for server-side query execution
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
@@ -1298,8 +1287,6 @@ impl NativeTable {
|
||||
params: Option<ReadParams>,
|
||||
read_consistency_interval: Option<std::time::Duration>,
|
||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
server_side_query_enabled: bool,
|
||||
managed_versioning: Option<bool>,
|
||||
) -> Result<Self> {
|
||||
let params = params.unwrap_or_default();
|
||||
// patch the params if we have a write store wrapper
|
||||
@@ -1308,54 +1295,17 @@ impl NativeTable {
|
||||
None => params,
|
||||
};
|
||||
|
||||
// Build table_id from namespace + name
|
||||
let mut table_id = namespace.clone();
|
||||
table_id.push(name.to_string());
|
||||
|
||||
// Determine if managed_versioning is enabled
|
||||
// Use the provided value if available, otherwise query the namespace
|
||||
let managed_versioning = match managed_versioning {
|
||||
Some(value) => value,
|
||||
None if namespace_client.is_some() => {
|
||||
let ns_client = namespace_client.as_ref().unwrap();
|
||||
let describe_request = DescribeTableRequest {
|
||||
id: Some(table_id.clone()),
|
||||
..Default::default()
|
||||
};
|
||||
let response = ns_client
|
||||
.describe_table(describe_request)
|
||||
.await
|
||||
.map_err(|e| Error::Runtime {
|
||||
message: format!(
|
||||
"Failed to describe table via namespace client: {}. \
|
||||
If you don't need managed versioning, don't pass namespace_client.",
|
||||
e
|
||||
),
|
||||
})?;
|
||||
response.managed_versioning == Some(true)
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
|
||||
let mut builder = DatasetBuilder::from_uri(uri).with_read_params(params);
|
||||
|
||||
// Set up commit handler when managed_versioning is enabled
|
||||
if managed_versioning && let Some(ref ns_client) = namespace_client {
|
||||
let external_store =
|
||||
LanceNamespaceExternalManifestStore::new(ns_client.clone(), table_id.clone());
|
||||
let commit_handler: Arc<dyn CommitHandler> = Arc::new(ExternalManifestCommitHandler {
|
||||
external_manifest_store: Arc::new(external_store),
|
||||
});
|
||||
builder = builder.with_commit_handler(commit_handler);
|
||||
}
|
||||
|
||||
let dataset = builder.load().await.map_err(|e| match e {
|
||||
lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
|
||||
name: name.to_string(),
|
||||
source: Box::new(e),
|
||||
},
|
||||
e => e.into(),
|
||||
})?;
|
||||
let dataset = DatasetBuilder::from_uri(uri)
|
||||
.with_read_params(params)
|
||||
.load()
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
lance::Error::DatasetNotFound { .. } => Error::TableNotFound {
|
||||
name: name.to_string(),
|
||||
source: Box::new(e),
|
||||
},
|
||||
e => e.into(),
|
||||
})?;
|
||||
|
||||
let dataset = DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval);
|
||||
let id = Self::build_id(&namespace, name);
|
||||
@@ -1368,7 +1318,6 @@ impl NativeTable {
|
||||
dataset,
|
||||
read_consistency_interval,
|
||||
namespace_client,
|
||||
server_side_query_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1472,7 +1421,6 @@ impl NativeTable {
|
||||
dataset,
|
||||
read_consistency_interval,
|
||||
namespace_client: stored_namespace_client,
|
||||
server_side_query_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1512,8 +1460,7 @@ impl NativeTable {
|
||||
/// * `namespace` - The namespace path. When non-empty, an explicit URI must be provided.
|
||||
/// * `batches` RecordBatch to be saved in the database.
|
||||
/// * `params` - Write parameters.
|
||||
/// * `namespace_client` - Optional namespace client for namespace operations
|
||||
/// * `server_side_query_enabled` - Whether to enable server-side query execution
|
||||
/// * `namespace_client` - Optional namespace client for server-side query execution
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
@@ -1528,7 +1475,6 @@ impl NativeTable {
|
||||
params: Option<WriteParams>,
|
||||
read_consistency_interval: Option<std::time::Duration>,
|
||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
server_side_query_enabled: bool,
|
||||
) -> Result<Self> {
|
||||
// Default params uses format v1.
|
||||
let params = params.unwrap_or(WriteParams {
|
||||
@@ -1561,7 +1507,6 @@ impl NativeTable {
|
||||
dataset: DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval),
|
||||
read_consistency_interval,
|
||||
namespace_client,
|
||||
server_side_query_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1575,7 +1520,6 @@ impl NativeTable {
|
||||
params: Option<WriteParams>,
|
||||
read_consistency_interval: Option<std::time::Duration>,
|
||||
namespace_client: Option<Arc<dyn LanceNamespace>>,
|
||||
server_side_query_enabled: bool,
|
||||
) -> Result<Self> {
|
||||
let data: Box<dyn Scannable> = Box::new(RecordBatch::new_empty(schema));
|
||||
Self::create(
|
||||
@@ -1587,7 +1531,6 @@ impl NativeTable {
|
||||
params,
|
||||
read_consistency_interval,
|
||||
namespace_client,
|
||||
server_side_query_enabled,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1691,7 +1634,6 @@ impl NativeTable {
|
||||
dataset: DatasetConsistencyWrapper::new_latest(dataset, read_consistency_interval),
|
||||
read_consistency_interval,
|
||||
namespace_client: stored_namespace_client,
|
||||
server_side_query_enabled,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2683,7 +2625,7 @@ mod tests {
|
||||
vec![Ok(batch.clone())],
|
||||
batch.schema(),
|
||||
));
|
||||
let table = NativeTable::create(uri, "test", vec![], reader, None, None, None, None, false)
|
||||
let table = NativeTable::create(uri, "test", vec![], reader, None, None, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow_cast::can_cast_types;
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
|
||||
use datafusion::functions::core::{get_field, named_struct};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datafusion_common::config::ConfigOptions;
|
||||
use datafusion_physical_expr::ScalarFunctionExpr;
|
||||
use datafusion_physical_expr::expressions::{CastExpr, Literal};
|
||||
use datafusion_physical_expr::expressions::{Literal, cast};
|
||||
use datafusion_physical_plan::expressions::Column;
|
||||
use datafusion_physical_plan::projection::ProjectionExec;
|
||||
use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr};
|
||||
@@ -26,9 +25,12 @@ pub fn cast_to_table_schema(
|
||||
return Ok(input);
|
||||
}
|
||||
|
||||
let exprs = build_field_exprs(input_schema.fields(), table_schema.fields(), &|idx| {
|
||||
Arc::new(Column::new(input_schema.field(idx).name(), idx)) as Arc<dyn PhysicalExpr>
|
||||
})?;
|
||||
let exprs = build_field_exprs(
|
||||
input_schema.fields(),
|
||||
table_schema.fields(),
|
||||
&|idx| Arc::new(Column::new(input_schema.field(idx).name(), idx)) as Arc<dyn PhysicalExpr>,
|
||||
&input_schema,
|
||||
)?;
|
||||
|
||||
let exprs: Vec<(Arc<dyn PhysicalExpr>, String)> = exprs
|
||||
.into_iter()
|
||||
@@ -49,6 +51,7 @@ fn build_field_exprs(
|
||||
input_fields: &Fields,
|
||||
table_fields: &Fields,
|
||||
get_input_expr: &dyn Fn(usize) -> Arc<dyn PhysicalExpr>,
|
||||
input_schema: &Schema,
|
||||
) -> Result<Vec<(Arc<dyn PhysicalExpr>, FieldRef)>> {
|
||||
let config = Arc::new(ConfigOptions::default());
|
||||
let mut result = Vec::new();
|
||||
@@ -69,19 +72,24 @@ fn build_field_exprs(
|
||||
(DataType::Struct(in_children), DataType::Struct(tbl_children))
|
||||
if in_children != tbl_children =>
|
||||
{
|
||||
let sub_exprs = build_field_exprs(in_children, tbl_children, &|child_idx| {
|
||||
let child_name = in_children[child_idx].name();
|
||||
Arc::new(ScalarFunctionExpr::new(
|
||||
&format!("get_field({child_name})"),
|
||||
get_field(),
|
||||
vec![
|
||||
input_expr.clone(),
|
||||
Arc::new(Literal::new(ScalarValue::from(child_name.as_str()))),
|
||||
],
|
||||
Arc::new(in_children[child_idx].as_ref().clone()),
|
||||
config.clone(),
|
||||
)) as Arc<dyn PhysicalExpr>
|
||||
})?;
|
||||
let sub_exprs = build_field_exprs(
|
||||
in_children,
|
||||
tbl_children,
|
||||
&|child_idx| {
|
||||
let child_name = in_children[child_idx].name();
|
||||
Arc::new(ScalarFunctionExpr::new(
|
||||
&format!("get_field({child_name})"),
|
||||
get_field(),
|
||||
vec![
|
||||
input_expr.clone(),
|
||||
Arc::new(Literal::new(ScalarValue::from(child_name.as_str()))),
|
||||
],
|
||||
Arc::new(in_children[child_idx].as_ref().clone()),
|
||||
config.clone(),
|
||||
)) as Arc<dyn PhysicalExpr>
|
||||
},
|
||||
input_schema,
|
||||
)?;
|
||||
|
||||
let output_struct_fields: Fields = sub_exprs
|
||||
.iter()
|
||||
@@ -117,21 +125,17 @@ fn build_field_exprs(
|
||||
// Types match: pass through.
|
||||
(inp, tbl) if inp == tbl => input_expr,
|
||||
// Types differ: cast.
|
||||
// safe: false (the default) means overflow/truncation errors surface at execution time.
|
||||
(_, _) if can_cast_types(input_field.data_type(), table_field.data_type()) => Arc::new(
|
||||
CastExpr::new(input_expr, table_field.data_type().clone(), None),
|
||||
)
|
||||
as Arc<dyn PhysicalExpr>,
|
||||
(inp, tbl) => {
|
||||
return Err(Error::InvalidInput {
|
||||
_ => cast(input_expr, input_schema, table_field.data_type().clone()).map_err(|e| {
|
||||
Error::InvalidInput {
|
||||
message: format!(
|
||||
"cannot cast field '{}' from {} to {}",
|
||||
"cannot cast field '{}' from {} to {}: {}",
|
||||
table_field.name(),
|
||||
inp,
|
||||
tbl,
|
||||
input_field.data_type(),
|
||||
table_field.data_type(),
|
||||
e
|
||||
),
|
||||
});
|
||||
}
|
||||
}
|
||||
})?,
|
||||
};
|
||||
|
||||
let output_field = Arc::new(Field::new(
|
||||
@@ -149,12 +153,10 @@ fn build_field_exprs(
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::buffer::OffsetBuffer;
|
||||
use arrow_array::{
|
||||
Array, Float32Array, Float64Array, Int32Array, Int64Array, ListArray, RecordBatch,
|
||||
StringArray, StructArray, UInt32Array, UInt64Array,
|
||||
Float32Array, Float64Array, Int32Array, Int64Array, RecordBatch, StringArray, StructArray,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Fields, Schema};
|
||||
use arrow_schema::{DataType, Field, Schema};
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datafusion_catalog::MemTable;
|
||||
use futures::TryStreamExt;
|
||||
@@ -493,129 +495,4 @@ mod tests {
|
||||
assert_eq!(b.value(0), "hello");
|
||||
assert_eq!(b.value(1), "world");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_narrowing_numeric_cast_success() {
|
||||
let input_batch = RecordBatch::try_new(
|
||||
Arc::new(Schema::new(vec![Field::new("a", DataType::UInt64, false)])),
|
||||
vec![Arc::new(UInt64Array::from(vec![1u64, 2, 3]))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let table_schema = Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
|
||||
|
||||
let plan = plan_from_batch(input_batch).await;
|
||||
let casted = cast_to_table_schema(plan, &table_schema).unwrap();
|
||||
let result = collect(casted).await;
|
||||
|
||||
assert_eq!(result.schema().field(0).data_type(), &DataType::UInt32);
|
||||
let a: &UInt32Array = result.column(0).as_any().downcast_ref().unwrap();
|
||||
assert_eq!(a.values(), &[1u32, 2, 3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_narrowing_numeric_cast_overflow_errors() {
|
||||
let overflow_val = u32::MAX as u64 + 1;
|
||||
let input_batch = RecordBatch::try_new(
|
||||
Arc::new(Schema::new(vec![Field::new("a", DataType::UInt64, false)])),
|
||||
vec![Arc::new(UInt64Array::from(vec![overflow_val]))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let table_schema = Schema::new(vec![Field::new("a", DataType::UInt32, false)]);
|
||||
|
||||
let plan = plan_from_batch(input_batch).await;
|
||||
// Planning succeeds — the overflow is only detected at execution time.
|
||||
let casted = cast_to_table_schema(plan, &table_schema).unwrap();
|
||||
|
||||
let ctx = SessionContext::new();
|
||||
let stream = casted.execute(0, ctx.task_ctx()).unwrap();
|
||||
let result: Result<Vec<RecordBatch>, _> = stream.try_collect().await;
|
||||
assert!(result.is_err(), "expected overflow error at execution time");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_list_struct_field_reorder() {
|
||||
// list<struct<a: Int32, b: Int32>> → list<struct<b: Int64, a: Int64>>
|
||||
// Tests both reordering (a,b → b,a) and element-type widening (Int32 → Int64).
|
||||
let inner_fields: Fields = vec![
|
||||
Field::new("a", DataType::Int32, true),
|
||||
Field::new("b", DataType::Int32, true),
|
||||
]
|
||||
.into();
|
||||
let struct_array = StructArray::from(vec![
|
||||
(
|
||||
Arc::new(inner_fields[0].as_ref().clone()),
|
||||
Arc::new(Int32Array::from(vec![1, 3])) as _,
|
||||
),
|
||||
(
|
||||
Arc::new(inner_fields[1].as_ref().clone()),
|
||||
Arc::new(Int32Array::from(vec![2, 4])) as _,
|
||||
),
|
||||
]);
|
||||
// Offsets: one list element containing two struct rows (0..2).
|
||||
let offsets = OffsetBuffer::from_lengths(vec![2]);
|
||||
let list_array = ListArray::try_new(
|
||||
Arc::new(Field::new("item", DataType::Struct(inner_fields), true)),
|
||||
offsets,
|
||||
Arc::new(struct_array),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
let input_batch = RecordBatch::try_new(
|
||||
Arc::new(Schema::new(vec![Field::new(
|
||||
"s_list",
|
||||
list_array.data_type().clone(),
|
||||
false,
|
||||
)])),
|
||||
vec![Arc::new(list_array)],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let table_inner: Fields = vec![
|
||||
Field::new("b", DataType::Int64, true),
|
||||
Field::new("a", DataType::Int64, true),
|
||||
]
|
||||
.into();
|
||||
let table_schema = Schema::new(vec![Field::new(
|
||||
"s_list",
|
||||
DataType::List(Arc::new(Field::new(
|
||||
"item",
|
||||
DataType::Struct(table_inner),
|
||||
true,
|
||||
))),
|
||||
false,
|
||||
)]);
|
||||
|
||||
let plan = plan_from_batch(input_batch).await;
|
||||
let casted = cast_to_table_schema(plan, &table_schema).unwrap();
|
||||
let result = collect(casted).await;
|
||||
|
||||
let list_col = result
|
||||
.column(0)
|
||||
.as_any()
|
||||
.downcast_ref::<ListArray>()
|
||||
.unwrap();
|
||||
let struct_col = list_col
|
||||
.values()
|
||||
.as_any()
|
||||
.downcast_ref::<StructArray>()
|
||||
.unwrap();
|
||||
assert_eq!(struct_col.num_columns(), 2);
|
||||
|
||||
let b: &Int64Array = struct_col
|
||||
.column_by_name("b")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref()
|
||||
.unwrap();
|
||||
assert_eq!(b.values(), &[2, 4]);
|
||||
let a: &Int64Array = struct_col
|
||||
.column_by_name("a")
|
||||
.unwrap()
|
||||
.as_any()
|
||||
.downcast_ref()
|
||||
.unwrap();
|
||||
assert_eq!(a.values(), &[1, 3]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use futures::FutureExt;
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -24,7 +23,7 @@ pub struct DeleteResult {
|
||||
pub(crate) async fn execute_delete(table: &NativeTable, predicate: &str) -> Result<DeleteResult> {
|
||||
table.dataset.ensure_mutable()?;
|
||||
let mut dataset = (*table.dataset.get().await?).clone();
|
||||
let delete_result = dataset.delete(predicate).boxed().await?;
|
||||
let delete_result = dataset.delete(predicate).await?;
|
||||
let num_deleted_rows = delete_result.num_deleted_rows;
|
||||
let version = dataset.version().version;
|
||||
table.dataset.update(dataset);
|
||||
|
||||
@@ -40,10 +40,8 @@ pub async fn execute_query(
|
||||
query: &AnyQuery,
|
||||
options: QueryExecutionOptions,
|
||||
) -> Result<DatasetRecordBatchStream> {
|
||||
// If server-side query is enabled and namespace client is configured, use server-side query execution
|
||||
if table.server_side_query_enabled
|
||||
&& let Some(ref namespace_client) = table.namespace_client
|
||||
{
|
||||
// If namespace client is configured, use server-side query execution
|
||||
if let Some(ref namespace_client) = table.namespace_client {
|
||||
return execute_namespace_query(table, namespace_client.clone(), query, options).await;
|
||||
}
|
||||
execute_generic_query(table, query, options).await
|
||||
|
||||
Reference in New Issue
Block a user