mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-30 00:45:37 +00:00
chore: update lance dependency to v13.0.0-beta.12 (#4265)
Update the Rust workspace Lance dependencies and Java lance-core from v13.0.0-beta.8 to [v13.0.0-beta.12](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.12), refreshing Cargo.lock. No compatibility fixes were required; validation passed with `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. --------- Co-authored-by: Lu Qiu <luqiujob@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
LanceDB Robot
co-authored by
Lu Qiu
Claude Fable 5.1
Lu Qiu
Claude Fable 5.1
parent
85daeb27cb
commit
53a2afdffa
Generated
+46
-44
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5216,8 +5216,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5239,7 +5239,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5253,7 +5253,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5262,8 +5262,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -5273,8 +5273,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5311,8 +5311,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5343,8 +5343,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5361,8 +5361,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5371,8 +5371,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5405,8 +5405,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5438,8 +5438,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-geo"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"datafusion",
|
||||
"geo-traits",
|
||||
@@ -5453,8 +5453,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5481,6 +5481,7 @@ dependencies = [
|
||||
"geoarrow-array",
|
||||
"geoarrow-schema",
|
||||
"half",
|
||||
"hex",
|
||||
"itertools 0.14.0",
|
||||
"jieba-rs",
|
||||
"jsonb",
|
||||
@@ -5518,12 +5519,13 @@ dependencies = [
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"twox-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5545,8 +5547,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5588,8 +5590,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5603,8 +5605,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5618,8 +5620,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5672,8 +5674,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5687,8 +5689,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5728,8 +5730,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5742,8 +5744,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "13.0.0-beta.8"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
|
||||
version = "13.0.0-beta.12"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
|
||||
+14
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lancedb = { path = "rust/lancedb", default-features = false }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<properties>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<arrow.version>15.0.0</arrow.version>
|
||||
<lance-core.version>13.0.0-beta.8</lance-core.version>
|
||||
<lance-core.version>13.0.0-beta.12</lance-core.version>
|
||||
<spotless.skip>false</spotless.skip>
|
||||
<spotless.version>2.30.0</spotless.version>
|
||||
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
|
||||
|
||||
@@ -771,12 +771,13 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
|
||||
if json_storage is not None:
|
||||
# Labelled through metadata rather than pa.json_(), which only exists on
|
||||
# newer PyArrow; Lance reads the extension name off the field either way.
|
||||
return pa.field(
|
||||
field.name,
|
||||
json_storage,
|
||||
field.nullable,
|
||||
{"ARROW:extension:name": "arrow.json"},
|
||||
)
|
||||
# The other metadata keys mirror the table field's: Lance swaps the name
|
||||
# back to lance.json on write and then requires the field to match the
|
||||
# stored one exactly, including the empty ``ARROW:extension:metadata``
|
||||
# that pyarrow records for a ``pa.json_()`` column.
|
||||
metadata = dict(target_field.metadata or {})
|
||||
metadata[b"ARROW:extension:name"] = b"arrow.json"
|
||||
return pa.field(field.name, json_storage, field.nullable, metadata)
|
||||
if pa.types.is_struct(target_field.type):
|
||||
if pa.types.is_struct(field.type):
|
||||
new_type = pa.struct(
|
||||
|
||||
@@ -948,6 +948,55 @@ def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
|
||||
]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_labels_json_text_with_table_extension_metadata():
|
||||
# A table created from pa.json_() stores the empty ARROW:extension:metadata that
|
||||
# pyarrow exports next to the name. Lance keeps that key when it turns the
|
||||
# arrow.json label back into lance.json on write and then requires the field to
|
||||
# match the stored one exactly, so the label has to mirror the table's metadata.
|
||||
target = pa.schema(
|
||||
[
|
||||
pa.field(
|
||||
"payload",
|
||||
pa.large_binary(),
|
||||
metadata={
|
||||
b"ARROW:extension:name": b"lance.json",
|
||||
b"ARROW:extension:metadata": b"",
|
||||
},
|
||||
),
|
||||
pa.field(
|
||||
"docs",
|
||||
pa.list_(
|
||||
pa.field(
|
||||
"item",
|
||||
pa.large_binary(),
|
||||
metadata={b"ARROW:extension:name": b"lance.json"},
|
||||
)
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
data = pa.table(
|
||||
{
|
||||
"payload": pa.array(['{"a": 1}', None], type=pa.string()),
|
||||
"docs": pa.array([['{"b": 2}'], []], type=pa.list_(pa.string())),
|
||||
}
|
||||
)
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
payload = output.schema.field("payload")
|
||||
assert payload.type == pa.string()
|
||||
assert payload.metadata == {
|
||||
b"ARROW:extension:name": b"arrow.json",
|
||||
b"ARROW:extension:metadata": b"",
|
||||
}
|
||||
docs_item = output.schema.field("docs").type.value_field
|
||||
assert docs_item.type == pa.string()
|
||||
assert docs_item.metadata == {b"ARROW:extension:name": b"arrow.json"}
|
||||
assert output["payload"].to_pylist() == ['{"a": 1}', None]
|
||||
assert output["docs"].to_pylist() == [['{"b": 2}'], []]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_coerces_nested_binary_blob():
|
||||
data = pa.table(
|
||||
{
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, LazyLock};
|
||||
|
||||
use arrow_array::StructArray;
|
||||
@@ -73,8 +72,12 @@ fn build_field_exprs(
|
||||
let input_expr = get_input_expr(input_idx);
|
||||
|
||||
// PyArrow's pa.json_() is already labelled arrow.json, which is what lance-core wants
|
||||
// to see, so pass it straight through.
|
||||
if is_json_field(table_field) && is_arrow_json_field(input_field) {
|
||||
// to see, so pass it straight through once its extension metadata matches the table's
|
||||
// (see `arrow_json_field`); otherwise it takes the relabelling cast below.
|
||||
if is_json_field(table_field)
|
||||
&& is_arrow_json_field(input_field)
|
||||
&& input_field.as_ref() == &arrow_json_field(table_field, input_field.data_type())
|
||||
{
|
||||
result.push((input_expr, Arc::clone(input_field) as FieldRef));
|
||||
continue;
|
||||
}
|
||||
@@ -299,11 +302,25 @@ fn arrow_json_storage_type(input: &DataType) -> Option<DataType> {
|
||||
}
|
||||
}
|
||||
|
||||
fn arrow_json_field(name: &str, storage: DataType, nullable: bool) -> Field {
|
||||
Field::new(name, storage, nullable).with_metadata(HashMap::from([(
|
||||
/// The arrow.json field lance-core should see for `table_field`, a lance.json leaf.
|
||||
///
|
||||
/// Lance-core converts the leaf to lance.json by swapping the extension name and keeping the
|
||||
/// rest of the field metadata, and the file writer then requires the result to match the stored
|
||||
/// field exactly, `ARROW:extension:metadata` included. PyArrow exports every extension field
|
||||
/// with that key (empty for JSON), while a schema built in Rust omits it, so the relabelled
|
||||
/// field mirrors the table field's metadata rather than carrying a fixed set of keys.
|
||||
fn arrow_json_field(table_field: &Field, storage: &DataType) -> Field {
|
||||
let mut metadata = table_field.metadata().clone();
|
||||
metadata.insert(
|
||||
ARROW_EXT_NAME_KEY.to_string(),
|
||||
ARROW_JSON_EXT_NAME.to_string(),
|
||||
)]))
|
||||
);
|
||||
Field::new(
|
||||
table_field.name(),
|
||||
storage.clone(),
|
||||
table_field.is_nullable(),
|
||||
)
|
||||
.with_metadata(metadata)
|
||||
}
|
||||
|
||||
/// Rewrite `table_field` so that every lance.json leaf the input supplies as text becomes an
|
||||
@@ -320,11 +337,7 @@ fn json_write_target(input_field: &Field, table_field: &Field) -> Option<Field>
|
||||
} else {
|
||||
arrow_json_storage_type(input_field.data_type())?
|
||||
};
|
||||
return Some(arrow_json_field(
|
||||
table_field.name(),
|
||||
storage,
|
||||
table_field.is_nullable(),
|
||||
));
|
||||
return Some(arrow_json_field(table_field, &storage));
|
||||
}
|
||||
|
||||
if !has_json_fields(table_field) {
|
||||
@@ -401,6 +414,7 @@ fn null_literal(field: &FieldRef) -> Result<Arc<dyn PhysicalExpr>> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::buffer::OffsetBuffer;
|
||||
@@ -408,7 +422,7 @@ mod tests {
|
||||
Array, Float32Array, Float64Array, Int32Array, Int64Array, ListArray, RecordBatch,
|
||||
StringArray, StructArray, UInt32Array, UInt64Array,
|
||||
};
|
||||
use arrow_schema::{DataType, Field, Fields, Schema};
|
||||
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
|
||||
use datafusion::prelude::SessionContext;
|
||||
use datafusion_catalog::MemTable;
|
||||
use futures::TryStreamExt;
|
||||
@@ -1007,7 +1021,7 @@ mod tests {
|
||||
)]);
|
||||
|
||||
let input_item = if input_labelled {
|
||||
Arc::new(arrow_json_field("item", item_type, true))
|
||||
Arc::new(arrow_json_field(&json_field("item", true), &item_type))
|
||||
} else {
|
||||
Arc::new(Field::new("item", item_type, true))
|
||||
};
|
||||
@@ -1318,4 +1332,150 @@ mod tests {
|
||||
assert_eq!(result.num_rows(), 2);
|
||||
assert_eq!(result.column(0).null_count(), 2);
|
||||
}
|
||||
|
||||
/// PyArrow records `ARROW:extension:metadata` (empty) alongside the extension name when it
|
||||
/// exports a `pa.json_()` column, so a table created from one stores that key. Lance-core
|
||||
/// swaps the name of a relabelled leaf back to lance.json but keeps the other metadata, and
|
||||
/// the file writer then requires the field to match the stored one key for key. The
|
||||
/// relabelled field must therefore mirror the table field's metadata.
|
||||
#[rstest::rstest]
|
||||
#[case::unlabelled_text(false)]
|
||||
#[case::arrow_json_without_metadata_key(true)]
|
||||
#[tokio::test]
|
||||
async fn test_relabelled_json_mirrors_table_extension_metadata(#[case] input_labelled: bool) {
|
||||
use lance_arrow::json::{ARROW_JSON_EXT_NAME, JSON_EXT_NAME};
|
||||
use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
|
||||
|
||||
let table_schema = Schema::new(vec![
|
||||
Field::new("data", DataType::LargeBinary, true).with_metadata(HashMap::from([
|
||||
(ARROW_EXT_NAME_KEY.to_string(), JSON_EXT_NAME.to_string()),
|
||||
(ARROW_EXT_META_KEY.to_string(), String::new()),
|
||||
])),
|
||||
]);
|
||||
|
||||
let mut input_field = Field::new("data", DataType::Utf8, true);
|
||||
if input_labelled {
|
||||
input_field = input_field.with_metadata(HashMap::from([(
|
||||
ARROW_EXT_NAME_KEY.to_string(),
|
||||
ARROW_JSON_EXT_NAME.to_string(),
|
||||
)]));
|
||||
}
|
||||
let input_schema = Arc::new(Schema::new(vec![input_field]));
|
||||
let values = StringArray::from(vec![Some(r#"{"x": 1}"#), None]);
|
||||
let input_batch = RecordBatch::try_new(input_schema, vec![Arc::new(values)]).unwrap();
|
||||
|
||||
let plan = plan_from_batch(input_batch).await;
|
||||
let projected = cast_to_table_schema(plan, &table_schema).unwrap();
|
||||
|
||||
let out_field = projected.schema().field_with_name("data").unwrap().clone();
|
||||
assert_eq!(out_field.data_type(), &DataType::Utf8);
|
||||
assert_eq!(
|
||||
out_field.metadata(),
|
||||
&HashMap::from([
|
||||
(
|
||||
ARROW_EXT_NAME_KEY.to_string(),
|
||||
ARROW_JSON_EXT_NAME.to_string()
|
||||
),
|
||||
(ARROW_EXT_META_KEY.to_string(), String::new()),
|
||||
]),
|
||||
"the arrow.json label must carry the table field's other extension metadata"
|
||||
);
|
||||
|
||||
let result = collect(projected).await;
|
||||
assert_eq!(result.num_rows(), 2);
|
||||
assert_eq!(result.column(0).null_count(), 1);
|
||||
}
|
||||
|
||||
/// The inverse: a table built in Rust records only the extension name, so an input that
|
||||
/// carries the empty metadata key (what pyarrow exports) must drop it, or lance-core sees a
|
||||
/// field the file schema does not describe.
|
||||
#[tokio::test]
|
||||
async fn test_arrow_json_input_drops_metadata_key_the_table_lacks() {
|
||||
use lance_arrow::json::{ARROW_JSON_EXT_NAME, json_field};
|
||||
use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
|
||||
|
||||
let table_schema = Schema::new(vec![json_field("data", true)]);
|
||||
|
||||
let input_field = Field::new("data", DataType::Utf8, true).with_metadata(HashMap::from([
|
||||
(
|
||||
ARROW_EXT_NAME_KEY.to_string(),
|
||||
ARROW_JSON_EXT_NAME.to_string(),
|
||||
),
|
||||
(ARROW_EXT_META_KEY.to_string(), String::new()),
|
||||
]));
|
||||
let input_schema = Arc::new(Schema::new(vec![input_field]));
|
||||
let values = StringArray::from(vec![Some(r#"{"x": 1}"#)]);
|
||||
let input_batch = RecordBatch::try_new(input_schema, vec![Arc::new(values)]).unwrap();
|
||||
|
||||
let plan = plan_from_batch(input_batch).await;
|
||||
let projected = cast_to_table_schema(plan, &table_schema).unwrap();
|
||||
|
||||
let out_field = projected.schema().field_with_name("data").unwrap().clone();
|
||||
assert_eq!(out_field.data_type(), &DataType::Utf8);
|
||||
assert_eq!(
|
||||
out_field.metadata(),
|
||||
&HashMap::from([(
|
||||
ARROW_EXT_NAME_KEY.to_string(),
|
||||
ARROW_JSON_EXT_NAME.to_string()
|
||||
)])
|
||||
);
|
||||
assert_eq!(collect(projected).await.num_rows(), 1);
|
||||
}
|
||||
|
||||
/// Nested leaves mirror their table leaf's metadata too, since lance-core checks each
|
||||
/// child of a list, map or struct against the stored child.
|
||||
#[tokio::test]
|
||||
async fn test_nested_relabelled_json_mirrors_table_extension_metadata() {
|
||||
use lance_arrow::json::{ARROW_JSON_EXT_NAME, JSON_EXT_NAME};
|
||||
use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
|
||||
|
||||
let table_item =
|
||||
Field::new("item", DataType::LargeBinary, true).with_metadata(HashMap::from([
|
||||
(ARROW_EXT_NAME_KEY.to_string(), JSON_EXT_NAME.to_string()),
|
||||
(ARROW_EXT_META_KEY.to_string(), String::new()),
|
||||
]));
|
||||
let table_schema = Schema::new(vec![Field::new(
|
||||
"docs",
|
||||
DataType::List(Arc::new(table_item)),
|
||||
true,
|
||||
)]);
|
||||
|
||||
let input_item: FieldRef = Arc::new(Field::new("item", DataType::Utf8, true));
|
||||
let input_schema = Arc::new(Schema::new(vec![Field::new(
|
||||
"docs",
|
||||
DataType::List(input_item.clone()),
|
||||
true,
|
||||
)]));
|
||||
let values = StringArray::from(vec![Some(r#"{"k": 1}"#), Some(r#"{"k": 2}"#)]);
|
||||
let input_batch = RecordBatch::try_new(
|
||||
input_schema,
|
||||
vec![Arc::new(ListArray::new(
|
||||
input_item,
|
||||
OffsetBuffer::new(vec![0, 1, 2].into()),
|
||||
Arc::new(values),
|
||||
None,
|
||||
))],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let plan = plan_from_batch(input_batch).await;
|
||||
let projected = cast_to_table_schema(plan, &table_schema).unwrap();
|
||||
|
||||
let out_field = projected.schema().field_with_name("docs").unwrap().clone();
|
||||
let DataType::List(out_item) = out_field.data_type() else {
|
||||
panic!("expected a list, got {}", out_field.data_type());
|
||||
};
|
||||
assert_eq!(out_item.data_type(), &DataType::Utf8);
|
||||
assert_eq!(
|
||||
out_item.metadata(),
|
||||
&HashMap::from([
|
||||
(
|
||||
ARROW_EXT_NAME_KEY.to_string(),
|
||||
ARROW_JSON_EXT_NAME.to_string()
|
||||
),
|
||||
(ARROW_EXT_META_KEY.to_string(), String::new()),
|
||||
])
|
||||
);
|
||||
assert_eq!(collect(projected).await.num_rows(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user