rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-12 23:02:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into gatekeeper/fix-4284-1

Browse Source
This commit is contained in:
Gatefixer
2026-09-24 00:10:03 +00:00
parent 30bf171171 85c9cf6240
commit 02666469fc
29 changed files with 697 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.40.0-beta.8"
current_version = "0.40.0-beta.9"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Cargo.lock
Generated
+49 -47
View File
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arc-swap",
"arrow",
@@ -5216,8 +5216,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5239,7 +5239,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5253,7 +5253,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5262,8 +5262,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrayref",
"crunchy",
@@ -5273,8 +5273,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5311,8 +5311,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow",
"arrow-array",
@@ -5343,8 +5343,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow",
"arrow-array",
@@ -5361,8 +5361,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"proc-macro2",
"quote",
@@ -5371,8 +5371,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5405,8 +5405,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5438,8 +5438,8 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"datafusion",
"geo-traits",
@@ -5453,8 +5453,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arc-swap",
"arrow",
@@ -5481,6 +5481,7 @@ dependencies = [
"geoarrow-array",
"geoarrow-schema",
"half",
"hex",
"itertools 0.14.0",
"jieba-rs",
"jsonb",
@@ -5518,12 +5519,13 @@ dependencies = [
"tempfile",
"tokio",
"tracing",
"twox-hash",
]
[[package]]
name = "lance-index-core"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5545,8 +5547,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow",
"arrow-array",
@@ -5588,8 +5590,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5603,8 +5605,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow",
"async-trait",
@@ -5618,8 +5620,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5672,8 +5674,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5687,8 +5689,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow",
"arrow-array",
@@ -5728,8 +5730,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5742,8 +5744,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "13.0.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.8#3c20b7875f621c929fdcecabe74dbca507ec0f0d"
version = "13.0.0-beta.12"
source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5756,7 +5758,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.40.0-beta.8"
version = "0.40.0-beta.9"
dependencies = [
"ahash",
"anyhow",
@@ -5853,7 +5855,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.40.0-beta.8"
version = "0.40.0-beta.9"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5878,7 +5880,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.40.0-beta.8"
version = "0.40.0-beta.9"
dependencies = [
"arc-swap",
"arrow",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.8", default-features = false, "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.8", "tag" = "v13.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.40.0-beta.8</version>
<version>0.40.0-beta.9</version>
</dependency>
```
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.8</version>
<version>0.40.0-beta.9</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.40.0-beta.8</version>
<version>0.40.0-beta.9</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>13.0.0-beta.8</lance-core.version>
<lance-core.version>13.0.0-beta.12</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.40.0-beta.8"
version = "0.40.0-beta.9"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/arrow.test.ts
+97
View File
@@ -601,6 +601,103 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
);
});
it.each([
["ArrayBuffer", new Uint8Array([104]).buffer],
["URL", new URL("https://example.com/")],
["empty object", {}],
[
"class instance with enumerable fields",
Object.assign(new Blob(["x"]), { position: 0 }),
],
])("rejects %s values without a schema", (_name, value) => {
expect(() => makeArrowTable([{ id: 1, value }])).toThrow(
/field value at row 0/,
);
expect(() => makeArrowTable([{ value }])).toThrow(
/field value at row 0/,
);
});
it.each([
["Binary", new Binary(), new Uint8Array([104]).buffer],
["Utf8", new Utf8(), new URL("https://example.com/")],
])("rejects %s values with a schema", (_name, type, value) => {
const schema = new Schema([new Field("value", type, true)]);
expect(() => makeArrowTable([{ value }], { schema })).toThrow(
/field value at row 0/,
);
});
it("rejects unsupported values in nested struct fields", () => {
const schema = new Schema([
new Field(
"metadata",
new Struct([new Field("bytes", new Binary(), true)]),
true,
),
]);
expect(() =>
makeArrowTable(
[{ metadata: { bytes: new Uint8Array([104]).buffer } }],
{ schema },
),
).toThrow(/field metadata\.bytes at row 0/);
});
it("rejects unsupported objects in list elements", () => {
const schema = new Schema([
new Field(
"images",
new List(new Field("item", new Binary(), true)),
true,
),
]);
expect(() =>
makeArrowTable([{ images: [new Blob(["abc"])] }], { schema }),
).toThrow(/field images\[0\] at row 0/);
});
it("rejects unsupported objects in nested lists", () => {
const schema = new Schema([
new Field(
"images",
new List(
new Field(
"item",
new List(new Field("item", new Binary(), true)),
true,
),
),
true,
),
]);
expect(() =>
makeArrowTable([{ images: [[new Blob(["abc"])]] }], { schema }),
).toThrow(/field images\[0\]\[0\] at row 0/);
});
it("rejects unsupported nested struct fields in lists", () => {
const schema = new Schema([
new Field(
"items",
new List(
new Field(
"item",
new Struct([new Field("bytes", new Binary(), true)]),
true,
),
),
true,
),
]);
expect(() =>
makeArrowTable(
[{ items: [{ bytes: new Uint8Array([104]).buffer }] }],
{ schema },
),
).toThrow(/field items\[0\]\.bytes at row 0/);
});
it("will ignore generated dictionary IDs when comparing inferred types", function () {
const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
dictionaryEncodeStrings: true,
nodejs/__test__/blob.test.ts
+21
View File
@@ -102,6 +102,27 @@ describe("BlobFile", () => {
});
describe("makeArrowTable blob columns", () => {
it.each([
["ArrayBuffer", new Uint8Array([104]).buffer],
["Blob", new Blob(["hello"])],
["File", new File(["hello"], "hello.txt")],
["URL", new URL("https://example.com/blob")],
["ReadableStream", new ReadableStream()],
["data struct with ArrayBuffer", { data: new Uint8Array([104]).buffer }],
["uri struct with URL", { uri: new URL("https://example.com/blob") }],
])("rejects %s input rather than writing a null blob", (_name, image) => {
const schema = new Schema([blob("image")]);
expect(() => makeArrowTable([{ image }], { schema })).toThrow(/image/);
});
it("rejects Blob instances with enumerable fields as one invalid value", () => {
const schema = new Schema([blob("image")]);
const image = Object.assign(new Blob(["hello"]), { position: 0 });
expect(() => makeArrowTable([{ image }], { schema })).toThrow(
"Unsupported object value for field image at row 0.",
);
});
it("coerces Buffer input onto a blob field", () => {
const schema = new Schema([
new Field("id", new Int64(), true),
nodejs/__test__/table.test.ts
+17
View File
@@ -2412,6 +2412,23 @@ describe("when dealing with blob columns", () => {
tmpDir.removeCallback();
});
it("rejects unsupported blob values in createTable and add", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
blob("image"),
]);
const row = { id: 1n, image: new Uint8Array([104]).buffer };
await expect(db.createTable("invalid", [row], { schema })).rejects.toThrow(
/field image at row 0/,
);
const table = await db.createEmptyTable("empty", schema);
await expect(table.add([row])).rejects.toThrow(/field image at row 0/);
await expect(table.countRows()).resolves.toBe(0);
});
it("discovers blob columns", async () => {
const { table } = await openBlobTable();
expect(await table.blobColumns()).toEqual(["image"]);
nodejs/lancedb/schema.ts
+49 -9
View File
@@ -62,6 +62,8 @@ class SchemaInferrer {
}
private observe(path: string[], value: unknown, row: number): void {
assertSupportedValue(value, path.join("."), row);
const current = this.fields.get(path);
if (current === undefined) {
this.addField(path, value, row);
@@ -430,19 +432,57 @@ function* recordPathsAndValues(
}
function isRecord(value: unknown): value is Record<string, unknown> {
if (typeof value !== "object" || value === null) {
return false;
}
const prototype = Object.getPrototypeOf(value);
return (
typeof value === "object" &&
value !== null &&
!Array.isArray(value) &&
!(value instanceof RegExp) &&
!(value instanceof Date) &&
!(value instanceof Set) &&
!(value instanceof Map) &&
!(value instanceof Buffer) &&
!ArrayBuffer.isView(value)
(prototype === Object.prototype || prototype === null) &&
Object.keys(value).length > 0
);
}
function isUnsupportedObject(value: unknown): boolean {
if (
typeof value !== "object" ||
value === null ||
Array.isArray(value) ||
isRecord(value)
) {
return false;
}
// Preserve the object types already passed through to Arrow. Other objects
// must not disappear while walking struct fields.
return !(
value instanceof RegExp ||
value instanceof Date ||
value instanceof Set ||
value instanceof Map ||
Buffer.isBuffer(value) ||
ArrayBuffer.isView(value)
);
}
function assertSupportedValue(
value: unknown,
field: string,
row: number,
): void {
if (Array.isArray(value)) {
for (const [index, item] of value.entries()) {
assertSupportedValue(item, `${field}[${index}]`, row);
}
} else if (isRecord(value)) {
for (const [name, child] of Object.entries(value)) {
assertSupportedValue(child, `${field}.${name}`, row);
}
} else if (isUnsupportedObject(value)) {
throw new Error(
`Unsupported object value for field ${field} at row ${row}.`,
);
}
}
function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
let fields = schema.fields;
let field: Field | undefined;
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.40.0-beta.8",
"version": "0.40.0-beta.9",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.40.0-beta.8"
version = "0.40.0-beta.9"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/table.py
+7 -6
View File
@@ -771,12 +771,13 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
if json_storage is not None:
# Labelled through metadata rather than pa.json_(), which only exists on
# newer PyArrow; Lance reads the extension name off the field either way.
return pa.field(
field.name,
json_storage,
field.nullable,
{"ARROW:extension:name": "arrow.json"},
)
# The other metadata keys mirror the table field's: Lance swaps the name
# back to lance.json on write and then requires the field to match the
# stored one exactly, including the empty ``ARROW:extension:metadata``
# that pyarrow records for a ``pa.json_()`` column.
metadata = dict(target_field.metadata or {})
metadata[b"ARROW:extension:name"] = b"arrow.json"
return pa.field(field.name, json_storage, field.nullable, metadata)
if pa.types.is_struct(target_field.type):
if pa.types.is_struct(field.type):
new_type = pa.struct(
python/python/tests/test_blob.py
+64
View File
@@ -999,6 +999,70 @@ def test_fetch_blob_files_lazy_read():
handles = table.fetch_blob_files("image", [by_id[1]])
assert len(handles) == 1
assert handles[0].read() == payload
handles[0].close()
assert handles[0].closed
with pytest.raises(RuntimeError, match="already closed"):
handles[0].read()
def test_blob_file_finalizer_and_close_on_runtime_worker():
# A tokio panic in IOBase.__del__ is printed but does not fail the add call.
# Run in a child process so the assertion catches that panic on stderr.
script = textwrap.dedent(
"""\
import gc
import threading
import lancedb
import pyarrow as pa
db = lancedb.connect("memory:///")
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
table = db.create_table("finalize", schema=schema)
table.add([{"id": 1, "image": b"first"}])
hits = table.search().with_row_id(True).limit(1).to_arrow()
row_id = hits["_rowid"][0].as_py()
gc_handle, explicit_handle = table.fetch_blob_files("image", [row_id, row_id])
gc.disable()
cycle = [gc_handle]
cycle.append(cycle)
del gc_handle, cycle
observed = []
def on_progress(_):
if not observed:
observed.append((
threading.current_thread() is not threading.main_thread(),
explicit_handle.closed,
gc.collect(),
))
explicit_handle.close()
table.add([{"id": 2, "image": b"second"}], progress=on_progress)
gc.enable()
assert len(observed) == 1, observed
assert observed[0][0] is True, observed
assert observed[0][1] is False, observed
assert observed[0][2] > 0, observed
assert explicit_handle.closed
explicit_handle.close() # Close remains idempotent after the worker call.
try:
explicit_handle.read()
except RuntimeError as error:
assert "already closed" in str(error)
else:
raise AssertionError("a closed blob file was readable")
print("worker close completed")
"""
)
result = subprocess.run(
[sys.executable, "-c", script], capture_output=True, text=True, check=False
)
assert result.returncode == 0, result.stderr
assert "Cannot start a runtime from within a runtime" not in result.stderr
assert "worker close completed" in result.stdout
def test_fetch_blob_files_null_alignment():
python/python/tests/test_remote_db.py
+4
View File
@@ -2470,6 +2470,10 @@ def test_remote_blob_files_are_lazy_seekable_handles():
assert alpha.read_range(1, 3) == b"lph"
gamma.seek(2)
assert gamma.read() == b"mma"
alpha.close()
assert alpha.closed
with pytest.raises(RuntimeError, match="already closed"):
alpha.read_range(0, 1)
def test_remote_blob_fetch_accepts_query_table():
python/python/tests/test_util.py
+49
View File
@@ -948,6 +948,55 @@ def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
]
def test_cast_to_target_schema_labels_json_text_with_table_extension_metadata():
# A table created from pa.json_() stores the empty ARROW:extension:metadata that
# pyarrow exports next to the name. Lance keeps that key when it turns the
# arrow.json label back into lance.json on write and then requires the field to
# match the stored one exactly, so the label has to mirror the table's metadata.
target = pa.schema(
[
pa.field(
"payload",
pa.large_binary(),
metadata={
b"ARROW:extension:name": b"lance.json",
b"ARROW:extension:metadata": b"",
},
),
pa.field(
"docs",
pa.list_(
pa.field(
"item",
pa.large_binary(),
metadata={b"ARROW:extension:name": b"lance.json"},
)
),
),
]
)
data = pa.table(
{
"payload": pa.array(['{"a": 1}', None], type=pa.string()),
"docs": pa.array([['{"b": 2}'], []], type=pa.list_(pa.string())),
}
)
output = _cast_to_target_schema(data.to_reader(), target).read_all()
payload = output.schema.field("payload")
assert payload.type == pa.string()
assert payload.metadata == {
b"ARROW:extension:name": b"arrow.json",
b"ARROW:extension:metadata": b"",
}
docs_item = output.schema.field("docs").type.value_field
assert docs_item.type == pa.string()
assert docs_item.metadata == {b"ARROW:extension:name": b"arrow.json"}
assert output["payload"].to_pylist() == ['{"a": 1}', None]
assert output["docs"].to_pylist() == [['{"b": 2}'], []]
def test_cast_to_target_schema_coerces_nested_binary_blob():
data = pa.table(
{
python/src/runtime.rs
+21
View File
@@ -197,6 +197,27 @@ pub fn block_on<F: std::future::Future>(fut: F) -> F::Output {
get_runtime().block_on(fut)
}
/// Run a detached task, including when the caller is already on a runtime worker.
/// Keep it visible to [`shutdown`] until it finishes.
pub fn spawn_background<F>(fut: F)
where
F: Future<Output = ()> + Send + 'static,
{
let guard = OutstandingGuard::new();
let task = async move {
let _guard = guard;
fut.await;
};
match runtime::Handle::try_current() {
Ok(handle) => {
let _ = handle.spawn(task);
}
Err(_) => {
let _ = get_runtime().spawn(task);
}
}
}
/// Gracefully quiesce the shared runtime, meant to run at normal process exit.
///
/// Waits (bounded by `timeout`) for [`OUTSTANDING`] to reach zero -- i.e.
python/src/table.rs
+40 -6
View File
@@ -1,8 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::{collections::HashMap, sync::Arc};
use std::{
collections::HashMap,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use crate::runtime::{block_on, future_into_py};
use crate::runtime::{block_on, future_into_py, spawn_background};
use crate::{
connection::Connection,
error::PythonErrorExt,
@@ -680,11 +686,23 @@ impl From<lancedb::table::DropColumnsResult> for DropColumnsResult {
#[pyclass(name = "BlobFile")]
pub struct PyBlobFile {
inner: Arc<BlobFile>,
closed: AtomicBool,
}
impl PyBlobFile {
fn ensure_open(&self) -> PyResult<()> {
if self.closed.load(Ordering::Acquire) {
Err(PyRuntimeError::new_err("blob file is already closed"))
} else {
Ok(())
}
}
}
#[pymethods]
impl PyBlobFile {
fn read_bytes(self_: PyRef<'_, Self>) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
@@ -694,6 +712,7 @@ impl PyBlobFile {
}
pub fn read(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let bytes = inner
@@ -705,7 +724,20 @@ impl PyBlobFile {
}
fn close(self_: PyRef<'_, Self>) -> PyResult<()> {
if self_.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
let inner = self_.inner.clone();
if tokio::runtime::Handle::try_current().is_ok() {
// IOBase.__del__ can call close while cyclic GC runs on a worker.
// The status changes immediately; release Lance's async state there.
spawn_background(async move {
if let Err(error) = inner.close().await {
log::warn!("blob close failed: {error}");
}
});
return Ok(());
}
self_
.py()
.detach(move || block_on(async move { inner.close().await }))
@@ -713,13 +745,11 @@ impl PyBlobFile {
}
fn is_closed(self_: PyRef<'_, Self>) -> bool {
let inner = self_.inner.clone();
self_
.py()
.detach(move || block_on(async move { inner.is_closed().await }))
self_.closed.load(Ordering::Acquire)
}
fn seek(self_: PyRef<'_, Self>, position: u64) -> PyResult<()> {
self_.ensure_open()?;
let inner = self_.inner.clone();
self_
.py()
@@ -728,6 +758,7 @@ impl PyBlobFile {
}
fn tell(self_: PyRef<'_, Self>) -> PyResult<u64> {
self_.ensure_open()?;
let inner = self_.inner.clone();
self_
.py()
@@ -741,6 +772,7 @@ impl PyBlobFile {
/// Read a blob-local byte range without moving the cursor.
fn read_range(self_: PyRef<'_, Self>, offset: u64, length: usize) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let end = offset
.checked_add(length as u64)
.ok_or_else(|| PyValueError::new_err("offset + length overflowed"))?;
@@ -753,6 +785,7 @@ impl PyBlobFile {
}
fn read_up_to(self_: PyRef<'_, Self>, length: usize) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
@@ -1471,6 +1504,7 @@ impl Table {
.map(|handle| {
handle.map(|file| PyBlobFile {
inner: Arc::new(file),
closed: AtomicBool::new(false),
})
})
.collect::<Vec<_>>())
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.40.0-beta.8"
version = "0.40.0-beta.9"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/remote/db.rs
+76 -31
View File
@@ -216,7 +216,8 @@ impl RemoteDatabaseOptionsBuilder {
#[derive(Debug)]
pub struct RemoteDatabase<S: HttpSend = Sender> {
client: RestfulLanceDbClient<S>,
table_cache: Cache<String, Arc<RemoteTable<S>>>,
// Cache existence and server capabilities, not mutable per-handle table state.
table_cache: Cache<String, ServerVersion>,
uri: String,
/// Headers to pass to the namespace client for authentication
namespace_headers: HashMap<String, String>,
@@ -1371,16 +1372,9 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
};
for table in &tables {
let table_identifier = build_table_identifier(table, &request.namespace_path)?;
build_table_identifier(table, &request.namespace_path)?;
let cache_key = build_cache_key(table, &request.namespace_path);
let remote_table = Arc::new(RemoteTable::new(
self.client.clone(),
table.clone(),
request.namespace_path.clone(),
table_identifier.clone(),
version.clone(),
));
self.table_cache.insert(cache_key, remote_table).await;
self.table_cache.insert(cache_key, version.clone()).await;
}
Ok(tables)
}
@@ -1407,16 +1401,9 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
// Cache the tables for future use
let namespace_vec = namespace_parts.to_vec();
for table in &response.tables {
let table_identifier = build_table_identifier(table, &namespace_vec)?;
build_table_identifier(table, &namespace_vec)?;
let cache_key = build_cache_key(table, &namespace_vec);
let remote_table = Arc::new(RemoteTable::new(
self.client.clone(),
table.clone(),
namespace_vec.clone(),
table_identifier.clone(),
version.clone(),
));
self.table_cache.insert(cache_key, remote_table).await;
self.table_cache.insert(cache_key, version.clone()).await;
}
Ok(response)
@@ -1484,9 +1471,9 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
request.name.clone(),
request.namespace_path.clone(),
table_identifier,
version,
version.clone(),
));
self.table_cache.insert(cache_key, table.clone()).await;
self.table_cache.insert(cache_key, version).await;
Ok(table)
}
@@ -1526,9 +1513,9 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
request.target_table_name.clone(),
request.target_namespace_path.clone(),
table_identifier,
version,
version.clone(),
));
self.table_cache.insert(cache_key, table.clone()).await;
self.table_cache.insert(cache_key, version).await;
Ok(table)
}
@@ -1537,10 +1524,17 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
let identifier = build_table_identifier(&request.name, &request.namespace_path)?;
let cache_key = build_cache_key(&request.name, &request.namespace_path);
// We describe the table to confirm it exists before moving on.
if let Some(table) = self.table_cache.get(&cache_key).await {
Ok(table.clone())
// Every open gets its own checkout, schema cache, and freshness state.
if let Some(version) = self.table_cache.get(&cache_key).await {
Ok(Arc::new(RemoteTable::new(
self.client.clone(),
request.name,
request.namespace_path,
identifier,
version,
)))
} else {
// Describe the table to confirm it exists before moving on.
let req = self
.client
.post(&format!("/v1/table/{}/describe/", identifier));
@@ -1550,13 +1544,12 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
let rsp = self.client.check_response(&request_id, rsp).await?;
let version = parse_server_version(&request_id, &rsp)?;
let describe_body = rsp.text().await.ok();
let table_identifier = build_table_identifier(&request.name, &request.namespace_path)?;
let table = Arc::new(RemoteTable::new(
self.client.clone(),
request.name.clone(),
request.namespace_path.clone(),
table_identifier,
version,
identifier,
version.clone(),
));
// This describe already carries the schema, so hand it to the table
// instead of making the first schema read fetch it again. A version or
@@ -1564,8 +1557,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
if let Some(body) = &describe_body {
table.seed_schema(body);
}
let cache_key = build_cache_key(&request.name, &request.namespace_path);
self.table_cache.insert(cache_key, table.clone()).await;
self.table_cache.insert(cache_key, version).await;
Ok(table)
}
}
@@ -2271,6 +2263,59 @@ mod tests {
assert_eq!(table.name(), "table1");
}
#[tokio::test]
async fn test_open_table_checkout_is_independent_per_handle() {
let latest = Arc::new(AtomicUsize::new(2));
let current = latest.clone();
let mut db = super::RemoteDatabase::new_mock(move |request| {
let body = match request.url().path() {
"/v1/table/table1/describe/" => {
let requested_version = request
.body()
.and_then(|body| body.as_bytes())
.and_then(|body| serde_json::from_slice::<serde_json::Value>(body).ok())
.and_then(|body| body["version"].as_u64());
let version =
requested_version.unwrap_or_else(|| current.load(Ordering::SeqCst) as u64);
serde_json::json!({
"version": version,
"schema": {"fields": [
{"name": "id", "type": {"type": "int32"}, "nullable": false}
]}
})
.to_string()
}
"/v1/table/table1/insert/" => {
let version = current.fetch_add(1, Ordering::SeqCst) + 1;
serde_json::json!({"version": version}).to_string()
}
path => panic!("unexpected request path: {path}"),
};
http::Response::builder().status(200).body(body).unwrap()
});
db.table_cache = moka::future::Cache::new(10);
let conn = Connection::new(
Arc::new(db),
Arc::new(crate::embeddings::MemoryRegistry::new()),
);
let writer = conn.open_table("table1").execute().await.unwrap();
let reader = conn.open_table("table1").execute().await.unwrap();
reader.checkout(1).await.unwrap();
assert_eq!(reader.version().await.unwrap(), 1);
assert_eq!(writer.version().await.unwrap(), 2);
let data = RecordBatch::try_new(
Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])),
vec![Arc::new(Int32Array::from(vec![3]))],
)
.unwrap();
assert_eq!(writer.add(data).execute().await.unwrap().version, 3);
assert_eq!(writer.version().await.unwrap(), 3);
assert_eq!(reader.version().await.unwrap(), 1);
}
#[tokio::test]
async fn test_open_table_seeds_the_schema_from_its_describe() {
let describe_calls = Arc::new(AtomicUsize::new(0));
rust/lancedb/src/table/datafusion/cast.rs
+173 -13
View File
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::collections::HashMap;
use std::sync::{Arc, LazyLock};
use arrow_array::StructArray;
@@ -73,8 +72,12 @@ fn build_field_exprs(
let input_expr = get_input_expr(input_idx);
// PyArrow's pa.json_() is already labelled arrow.json, which is what lance-core wants
// to see, so pass it straight through.
if is_json_field(table_field) && is_arrow_json_field(input_field) {
// to see, so pass it straight through once its extension metadata matches the table's
// (see `arrow_json_field`); otherwise it takes the relabelling cast below.
if is_json_field(table_field)
&& is_arrow_json_field(input_field)
&& input_field.as_ref() == &arrow_json_field(table_field, input_field.data_type())
{
result.push((input_expr, Arc::clone(input_field) as FieldRef));
continue;
}
@@ -299,11 +302,25 @@ fn arrow_json_storage_type(input: &DataType) -> Option<DataType> {
}
}
fn arrow_json_field(name: &str, storage: DataType, nullable: bool) -> Field {
Field::new(name, storage, nullable).with_metadata(HashMap::from([(
/// The arrow.json field lance-core should see for `table_field`, a lance.json leaf.
///
/// Lance-core converts the leaf to lance.json by swapping the extension name and keeping the
/// rest of the field metadata, and the file writer then requires the result to match the stored
/// field exactly, `ARROW:extension:metadata` included. PyArrow exports every extension field
/// with that key (empty for JSON), while a schema built in Rust omits it, so the relabelled
/// field mirrors the table field's metadata rather than carrying a fixed set of keys.
fn arrow_json_field(table_field: &Field, storage: &DataType) -> Field {
let mut metadata = table_field.metadata().clone();
metadata.insert(
ARROW_EXT_NAME_KEY.to_string(),
ARROW_JSON_EXT_NAME.to_string(),
)]))
);
Field::new(
table_field.name(),
storage.clone(),
table_field.is_nullable(),
)
.with_metadata(metadata)
}
/// Rewrite `table_field` so that every lance.json leaf the input supplies as text becomes an
@@ -320,11 +337,7 @@ fn json_write_target(input_field: &Field, table_field: &Field) -> Option<Field>
} else {
arrow_json_storage_type(input_field.data_type())?
};
return Some(arrow_json_field(
table_field.name(),
storage,
table_field.is_nullable(),
));
return Some(arrow_json_field(table_field, &storage));
}
if !has_json_fields(table_field) {
@@ -401,6 +414,7 @@ fn null_literal(field: &FieldRef) -> Result<Arc<dyn PhysicalExpr>> {
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use arrow::buffer::OffsetBuffer;
@@ -408,7 +422,7 @@ mod tests {
Array, Float32Array, Float64Array, Int32Array, Int64Array, ListArray, RecordBatch,
StringArray, StructArray, UInt32Array, UInt64Array,
};
use arrow_schema::{DataType, Field, Fields, Schema};
use arrow_schema::{DataType, Field, FieldRef, Fields, Schema};
use datafusion::prelude::SessionContext;
use datafusion_catalog::MemTable;
use futures::TryStreamExt;
@@ -1007,7 +1021,7 @@ mod tests {
)]);
let input_item = if input_labelled {
Arc::new(arrow_json_field("item", item_type, true))
Arc::new(arrow_json_field(&json_field("item", true), &item_type))
} else {
Arc::new(Field::new("item", item_type, true))
};
@@ -1318,4 +1332,150 @@ mod tests {
assert_eq!(result.num_rows(), 2);
assert_eq!(result.column(0).null_count(), 2);
}
/// PyArrow records `ARROW:extension:metadata` (empty) alongside the extension name when it
/// exports a `pa.json_()` column, so a table created from one stores that key. Lance-core
/// swaps the name of a relabelled leaf back to lance.json but keeps the other metadata, and
/// the file writer then requires the field to match the stored one key for key. The
/// relabelled field must therefore mirror the table field's metadata.
#[rstest::rstest]
#[case::unlabelled_text(false)]
#[case::arrow_json_without_metadata_key(true)]
#[tokio::test]
async fn test_relabelled_json_mirrors_table_extension_metadata(#[case] input_labelled: bool) {
use lance_arrow::json::{ARROW_JSON_EXT_NAME, JSON_EXT_NAME};
use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
let table_schema = Schema::new(vec![
Field::new("data", DataType::LargeBinary, true).with_metadata(HashMap::from([
(ARROW_EXT_NAME_KEY.to_string(), JSON_EXT_NAME.to_string()),
(ARROW_EXT_META_KEY.to_string(), String::new()),
])),
]);
let mut input_field = Field::new("data", DataType::Utf8, true);
if input_labelled {
input_field = input_field.with_metadata(HashMap::from([(
ARROW_EXT_NAME_KEY.to_string(),
ARROW_JSON_EXT_NAME.to_string(),
)]));
}
let input_schema = Arc::new(Schema::new(vec![input_field]));
let values = StringArray::from(vec![Some(r#"{"x": 1}"#), None]);
let input_batch = RecordBatch::try_new(input_schema, vec![Arc::new(values)]).unwrap();
let plan = plan_from_batch(input_batch).await;
let projected = cast_to_table_schema(plan, &table_schema).unwrap();
let out_field = projected.schema().field_with_name("data").unwrap().clone();
assert_eq!(out_field.data_type(), &DataType::Utf8);
assert_eq!(
out_field.metadata(),
&HashMap::from([
(
ARROW_EXT_NAME_KEY.to_string(),
ARROW_JSON_EXT_NAME.to_string()
),
(ARROW_EXT_META_KEY.to_string(), String::new()),
]),
"the arrow.json label must carry the table field's other extension metadata"
);
let result = collect(projected).await;
assert_eq!(result.num_rows(), 2);
assert_eq!(result.column(0).null_count(), 1);
}
/// The inverse: a table built in Rust records only the extension name, so an input that
/// carries the empty metadata key (what pyarrow exports) must drop it, or lance-core sees a
/// field the file schema does not describe.
#[tokio::test]
async fn test_arrow_json_input_drops_metadata_key_the_table_lacks() {
use lance_arrow::json::{ARROW_JSON_EXT_NAME, json_field};
use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
let table_schema = Schema::new(vec![json_field("data", true)]);
let input_field = Field::new("data", DataType::Utf8, true).with_metadata(HashMap::from([
(
ARROW_EXT_NAME_KEY.to_string(),
ARROW_JSON_EXT_NAME.to_string(),
),
(ARROW_EXT_META_KEY.to_string(), String::new()),
]));
let input_schema = Arc::new(Schema::new(vec![input_field]));
let values = StringArray::from(vec![Some(r#"{"x": 1}"#)]);
let input_batch = RecordBatch::try_new(input_schema, vec![Arc::new(values)]).unwrap();
let plan = plan_from_batch(input_batch).await;
let projected = cast_to_table_schema(plan, &table_schema).unwrap();
let out_field = projected.schema().field_with_name("data").unwrap().clone();
assert_eq!(out_field.data_type(), &DataType::Utf8);
assert_eq!(
out_field.metadata(),
&HashMap::from([(
ARROW_EXT_NAME_KEY.to_string(),
ARROW_JSON_EXT_NAME.to_string()
)])
);
assert_eq!(collect(projected).await.num_rows(), 1);
}
/// Nested leaves mirror their table leaf's metadata too, since lance-core checks each
/// child of a list, map or struct against the stored child.
#[tokio::test]
async fn test_nested_relabelled_json_mirrors_table_extension_metadata() {
use lance_arrow::json::{ARROW_JSON_EXT_NAME, JSON_EXT_NAME};
use lance_arrow::{ARROW_EXT_META_KEY, ARROW_EXT_NAME_KEY};
let table_item =
Field::new("item", DataType::LargeBinary, true).with_metadata(HashMap::from([
(ARROW_EXT_NAME_KEY.to_string(), JSON_EXT_NAME.to_string()),
(ARROW_EXT_META_KEY.to_string(), String::new()),
]));
let table_schema = Schema::new(vec![Field::new(
"docs",
DataType::List(Arc::new(table_item)),
true,
)]);
let input_item: FieldRef = Arc::new(Field::new("item", DataType::Utf8, true));
let input_schema = Arc::new(Schema::new(vec![Field::new(
"docs",
DataType::List(input_item.clone()),
true,
)]));
let values = StringArray::from(vec![Some(r#"{"k": 1}"#), Some(r#"{"k": 2}"#)]);
let input_batch = RecordBatch::try_new(
input_schema,
vec![Arc::new(ListArray::new(
input_item,
OffsetBuffer::new(vec![0, 1, 2].into()),
Arc::new(values),
None,
))],
)
.unwrap();
let plan = plan_from_batch(input_batch).await;
let projected = cast_to_table_schema(plan, &table_schema).unwrap();
let out_field = projected.schema().field_with_name("docs").unwrap().clone();
let DataType::List(out_item) = out_field.data_type() else {
panic!("expected a list, got {}", out_field.data_type());
};
assert_eq!(out_item.data_type(), &DataType::Utf8);
assert_eq!(
out_item.metadata(),
&HashMap::from([
(
ARROW_EXT_NAME_KEY.to_string(),
ARROW_JSON_EXT_NAME.to_string()
),
(ARROW_EXT_META_KEY.to_string(), String::new()),
])
);
assert_eq!(collect(projected).await.num_rows(), 2);
}
}