mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-30 00:45:37 +00:00
chore: update lance dependency to v13.0.0-beta.12 (#4265)
Update the Rust workspace Lance dependencies and Java lance-core from v13.0.0-beta.8 to [v13.0.0-beta.12](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.12), refreshing Cargo.lock. No compatibility fixes were required; validation passed with `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. --------- Co-authored-by: Lu Qiu <luqiujob@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Lu Qiu
Claude Fable 5.1
parent
85daeb27cb
commit
53a2afdffa
@@ -771,12 +771,13 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
|
||||
if json_storage is not None:
|
||||
# Labelled through metadata rather than pa.json_(), which only exists on
|
||||
# newer PyArrow; Lance reads the extension name off the field either way.
|
||||
return pa.field(
|
||||
field.name,
|
||||
json_storage,
|
||||
field.nullable,
|
||||
{"ARROW:extension:name": "arrow.json"},
|
||||
)
|
||||
# The other metadata keys mirror the table field's: Lance swaps the name
|
||||
# back to lance.json on write and then requires the field to match the
|
||||
# stored one exactly, including the empty ``ARROW:extension:metadata``
|
||||
# that pyarrow records for a ``pa.json_()`` column.
|
||||
metadata = dict(target_field.metadata or {})
|
||||
metadata[b"ARROW:extension:name"] = b"arrow.json"
|
||||
return pa.field(field.name, json_storage, field.nullable, metadata)
|
||||
if pa.types.is_struct(target_field.type):
|
||||
if pa.types.is_struct(field.type):
|
||||
new_type = pa.struct(
|
||||
|
||||
@@ -948,6 +948,55 @@ def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
|
||||
]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_labels_json_text_with_table_extension_metadata():
|
||||
# A table created from pa.json_() stores the empty ARROW:extension:metadata that
|
||||
# pyarrow exports next to the name. Lance keeps that key when it turns the
|
||||
# arrow.json label back into lance.json on write and then requires the field to
|
||||
# match the stored one exactly, so the label has to mirror the table's metadata.
|
||||
target = pa.schema(
|
||||
[
|
||||
pa.field(
|
||||
"payload",
|
||||
pa.large_binary(),
|
||||
metadata={
|
||||
b"ARROW:extension:name": b"lance.json",
|
||||
b"ARROW:extension:metadata": b"",
|
||||
},
|
||||
),
|
||||
pa.field(
|
||||
"docs",
|
||||
pa.list_(
|
||||
pa.field(
|
||||
"item",
|
||||
pa.large_binary(),
|
||||
metadata={b"ARROW:extension:name": b"lance.json"},
|
||||
)
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
data = pa.table(
|
||||
{
|
||||
"payload": pa.array(['{"a": 1}', None], type=pa.string()),
|
||||
"docs": pa.array([['{"b": 2}'], []], type=pa.list_(pa.string())),
|
||||
}
|
||||
)
|
||||
|
||||
output = _cast_to_target_schema(data.to_reader(), target).read_all()
|
||||
|
||||
payload = output.schema.field("payload")
|
||||
assert payload.type == pa.string()
|
||||
assert payload.metadata == {
|
||||
b"ARROW:extension:name": b"arrow.json",
|
||||
b"ARROW:extension:metadata": b"",
|
||||
}
|
||||
docs_item = output.schema.field("docs").type.value_field
|
||||
assert docs_item.type == pa.string()
|
||||
assert docs_item.metadata == {b"ARROW:extension:name": b"arrow.json"}
|
||||
assert output["payload"].to_pylist() == ['{"a": 1}', None]
|
||||
assert output["docs"].to_pylist() == [['{"b": 2}'], []]
|
||||
|
||||
|
||||
def test_cast_to_target_schema_coerces_nested_binary_blob():
|
||||
data = pa.table(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user